diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 69192299a3f..00000000000 --- a/.flake8 +++ /dev/null @@ -1,6 +0,0 @@ -[flake8] -# Ignore some errors, since we autoformat them away already wherever possible -# from https://github.com/psf/black/blob/main/.flake8 -# E302 is ignored to support jupytext files -ignore = E203, E266, E501, W503, E302 -exclude = .ipynb_checkpoints,*_cookiecutter,cookiecutter,etl/steps/archive,etl/snapshots/archive diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index a30fe02e553..69b40e97c36 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -9,3 +9,6 @@ ab772adce9a987639cb1cde15b9a62a349568582 # switching to ruff 5647751506f0685c6704aebc31a59632904d0741 + +# update ruff +38d649354920a5bd549a4e1ac02185882d13aa60 diff --git a/.vscode/settings.json b/.vscode/settings.json index 666dc3fa9a5..cb06b055745 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -37,17 +37,11 @@ "**/docs/architecture/*.md" ], "files.exclude": { - "etl/steps/archive": true, - "snapshots/archive": true, "**/dataset_*_config.json": true, "**/dataset_*_values.json": true, "**/dataset_*.json.dvc": true, "**/dataset_*.feather.dvc": true }, - "search.exclude": { - "etl/steps/archive": true, - "snapshots/archive": true - }, "yaml.format.printWidth": 999, "ruff.path": [ ".venv/bin/ruff" diff --git a/apps/chart_animation/cli.py b/apps/chart_animation/cli.py index 75f8b4fbe05..67dd88e5425 100644 --- a/apps/chart_animation/cli.py +++ b/apps/chart_animation/cli.py @@ -1,6 +1,5 @@ -"""Create a GIF or video for a given chart URL. +"""Create a GIF or video for a given chart URL.""" -""" from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from urllib.parse import parse_qs, urlencode, urlparse, urlunparse diff --git a/apps/cli/__init__.py b/apps/cli/__init__.py index 685b57ab2d1..38007973251 100644 --- a/apps/cli/__init__.py +++ b/apps/cli/__init__.py @@ -167,7 +167,8 @@ def cli_back() -> None: "update": "apps.step_update.cli.cli", "archive": "apps.step_update.cli.archive_cli", "explorer-update": "apps.explorer_update.cli.cli", - "pr": "apps.utils.draft_pull_request.cli", + "prr": "apps.utils.draft_pull_request.cli", + "pr": "apps.pr.cli.cli", }, }, { diff --git a/apps/metadata_migrate/cli.py b/apps/metadata_migrate/cli.py index b096289a8c2..9ea6482ff38 100644 --- a/apps/metadata_migrate/cli.py +++ b/apps/metadata_migrate/cli.py @@ -209,9 +209,9 @@ def cli( # empty fields to be filled vars[col]["presentation"] = {} - vars[col]["presentation"][ - "title_public" - ] = "TBD - Indicator title to be shown in data pages, that overrides the indicator's title." + vars[col]["presentation"]["title_public"] = ( + "TBD - Indicator title to be shown in data pages, that overrides the indicator's title." + ) vars[col]["presentation"]["title_variant"] = "TBD - Indicator's title variant" vars[col]["presentation"]["attribution_short"] = "TBD - Indicator's attribution (shorter version)" vars[col]["presentation"]["faqs"] = [ diff --git a/apps/metagpt/cli.py b/apps/metagpt/cli.py index f2cf33c96a0..29869f1a9d3 100644 --- a/apps/metagpt/cli.py +++ b/apps/metagpt/cli.py @@ -1,4 +1,5 @@ """Client module.""" + import os from typing import Any, Dict, Literal, cast diff --git a/apps/metagpt/prompts.py b/apps/metagpt/prompts.py index 785dcd462f7..cd34dd41cfd 100644 --- a/apps/metagpt/prompts.py +++ b/apps/metagpt/prompts.py @@ -2,6 +2,7 @@ Contains instructions to correctly query chat GPT for the different use cases (snapshot, garden, grapher, etc.). This includes details on what each metadata field means. """ + import json from typing import Any, Dict, List diff --git a/apps/metagpt/utils.py b/apps/metagpt/utils.py index 6f2615008ac..e233d453217 100644 --- a/apps/metagpt/utils.py +++ b/apps/metagpt/utils.py @@ -1,4 +1,5 @@ """Auxiliary utils for metagpt app.""" + from pathlib import Path from typing import Dict, List diff --git a/apps/owidbot/cli.py b/apps/owidbot/cli.py index 3f597b8158f..3db6c9ec83b 100644 --- a/apps/owidbot/cli.py +++ b/apps/owidbot/cli.py @@ -9,7 +9,7 @@ from rich_click.rich_command import RichCommand from apps.owidbot import anomalist, chart_diff, data_diff, grapher -from etl.config import get_container_name +from etl.config import OWIDBOT_ACCESS_TOKEN, get_container_name from . import github_utils as gh_utils @@ -57,7 +57,7 @@ def cli( if repo_name not in get_args(REPOS): raise AssertionError("Invalid repo") - repo = gh_utils.get_repo(repo_name) + repo = gh_utils.get_repo(repo_name, access_token=OWIDBOT_ACCESS_TOKEN) pr = gh_utils.get_pr(repo, branch) if pr is None: log.warning(f"No open PR found for branch {branch}") diff --git a/apps/owidbot/github_utils.py b/apps/owidbot/github_utils.py index 082824a2841..abd5f138ab1 100644 --- a/apps/owidbot/github_utils.py +++ b/apps/owidbot/github_utils.py @@ -17,9 +17,12 @@ def get_repo(repo_name: str, access_token: Optional[str] = None) -> github.Repository.Repository: if not access_token: - assert config.OWIDBOT_ACCESS_TOKEN, "OWIDBOT_ACCESS_TOKEN is not set" - access_token = config.OWIDBOT_ACCESS_TOKEN - auth = Auth.Token(access_token) + # Don't auth, be aware that you won't be able to do write operations. You should + # set up your access token on https://github.com/settings/tokens. + auth = None + else: + auth = Auth.Token(access_token) + g = Github(auth=auth) return g.get_repo(f"owid/{repo_name}") @@ -96,6 +99,37 @@ def compute_git_blob_sha1(content: bytes) -> str: return sha1.hexdigest() +def _github_access_token(): + # Use GITHUB_TOKEN if set, otherwise use OWIDBOT_ACCESS_TOKEN + if config.GITHUB_TOKEN: + return config.GITHUB_TOKEN + elif config.OWIDBOT_ACCESS_TOKEN: + return config.OWIDBOT_ACCESS_TOKEN + else: + raise AssertionError("You need to set GITHUB_TOKEN or OWIDBOT_ACCESS_TOKEN in your .env file to commit.") + + +def create_branch_if_not_exists(repo_name: str, branch: str, dry_run: bool) -> None: + """Create a branch if it doesn't exist.""" + repo = get_repo(repo_name, access_token=_github_access_token()) + try: + repo.get_branch(branch) + except github.GithubException as e: + if e.status == 404: + if not dry_run: + try: + master_ref = repo.get_branch("main").commit.sha + log.info(f"Using 'main' branch as reference for creating {branch}.") + except github.GithubException: + master_ref = repo.get_branch("master").commit.sha + log.info(f"Using 'master' branch as reference for creating {branch}.") + log.info(f"Creating branch {branch} with reference {master_ref}.") + repo.create_git_ref(ref=f"refs/heads/{branch}", sha=master_ref) + log.info(f"Branch {branch} created in {repo.name}.") + else: + raise e + + def commit_file_to_github( content: str, repo_name: str, @@ -106,7 +140,7 @@ def commit_file_to_github( ) -> None: """Commit a table to a GitHub repository using the GitHub API.""" # Get the repository object - repo = get_repo(repo_name) + repo = get_repo(repo_name, access_token=_github_access_token()) new_content_checksum = compute_git_blob_sha1(content.encode("utf-8")) try: diff --git a/apps/pr/__init__.py b/apps/pr/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/apps/pr/categories.py b/apps/pr/categories.py new file mode 100644 index 00000000000..d5d9a5fc487 --- /dev/null +++ b/apps/pr/categories.py @@ -0,0 +1,69 @@ +# Add EMOJIs for each PR type +PR_CATEGORIES = { + "data": { + "emoji": "📊", + "emoji_raw": ":bar_chart:", + "description": "data update or addition", + }, + "bug": { + "emoji": "🐛", + "emoji_raw": ":bug:", + "description": "bug fix for the user", + }, + "refactor": { + "emoji": "🔨", + "emoji_raw": ":hammer:", + "description": "a code change that neither fixes a bug nor adds a feature for the user", + }, + "enhance": { + "emoji": "✨", + "emoji_raw": ":sparkles:", + "description": "visible improvement over a current implementation without adding a new feature or fixing a bug", + }, + "feature": { + "emoji": "🎉", + "emoji_raw": ":tada:", + "description": "new feature for the user", + }, + "docs": { + "emoji": "📜", + "emoji_raw": ":scroll:", + "description": "documentation only changes", + "shortcut_key": "0", + }, + "chore": { + "emoji": "🐝", + "emoji_raw": ":honeybee:", + "description": "upgrading dependencies, tooling, etc. No production code change", + }, + "style": { + "emoji": "💄", + "emoji_raw": ":lipstick:", + "description": "formatting, missing semi colons, etc. No production code change", + }, + "wip": { + "emoji": "🚧", + "emoji_raw": ":construction:", + "description": "work in progress - intermediate commits that will be explained later on", + }, + "tests": { + "emoji": "✅", + "emoji_raw": ":white_check_mark:", + "description": "adding missing tests, refactoring tests, etc. No production code change", + }, +} +PR_CATEGORIES_MD_DESCRIPTION = "- " + "\n- ".join( + f"**{choice}**: {choice_params['description']}" for choice, choice_params in PR_CATEGORIES.items() +) +PR_CATEGORIES_CHOICES = [ + { + "title": f"{v['emoji']} {k}", + "value": k, + "shortcut_key": v.get("shortcut_key", k[0]), + } + for k, v in PR_CATEGORIES.items() +] +PR_CATEGORIES_CHOICES = sorted(PR_CATEGORIES_CHOICES, key=lambda x: x["shortcut_key"]) +assert len(set([x["shortcut_key"].lower() for x in PR_CATEGORIES_CHOICES])) == len( + PR_CATEGORIES_CHOICES +), "Shortcut keys must be unique" diff --git a/apps/pr/cli.py b/apps/pr/cli.py new file mode 100644 index 00000000000..3de16f52888 --- /dev/null +++ b/apps/pr/cli.py @@ -0,0 +1,445 @@ +"""This script creates a new draft pull request in GitHub, which starts a new staging server. + +Arguments: + +`TITLE`: The title of the PR. This must be given. + +`CATEGORY`: The category of the PR. This is optional. If not given, the user will be prompted to choose one. + +**Main use case**: Branch out from `master` to a temporary `work_branch`, and create a PR to merge `work_branch` -> `master`. You will be asked to choose a category. The value of `work_branch` will be auto-generated based on the title and the category. + +```shell +# Without specifying a category (you will be prompted for a category) +etl pr "some title for the PR" + +# With a category +etl pr "some title for the PR" data + +# With private stating server +etl pr "some title for the PR" --private +``` + +**Custom use case (1)**: Same as main use case, but with a specific branch name for the `work_branch`. + +```shell +etl pr "some title for the PR" --work-branch "this-temporary-branch" +# Shorter +etl pr "some title for the PR" -w "this-temporary-branch" +``` + +**Custom use case (2)**: Create a pull request from `current_branch` to `master`. + +```shell +etl pr "some title for the PR" --direct +``` + +**Custom use case (3)**: Create a pull request from branch `this-temporary-branch` -> `develop`. + +```shell +etl pr "some title for the PR" --direct --base-branch "develop" --work-branch "this-temporary-branch" +# Shorter +etl pr "some title for the PR" --direct -b "develop" -w "this-temporary-branch" +``` +""" + +import hashlib +import os +import re +import uuid +from typing import Optional, cast + +import click +import questionary +import requests +from git import GitCommandError, Repo +from rich_click.rich_command import RichCommand +from structlog import get_logger + +from apps.pr.categories import PR_CATEGORIES, PR_CATEGORIES_CHOICES +from apps.utils.gpt import OpenAIWrapper +from etl.config import GITHUB_TOKEN +from etl.paths import BASE_DIR + +# Initialize logger. +log = get_logger() + +# URL of the Github API, to be used to create a draft pull request in the ETL repos. +GITHUB_API_URL = "https://api.github.com/repos/owid/etl/pulls" + +# Style for questionary +SHELL_FORM_STYLE = questionary.Style( + [ + ("qmark", "fg:#fac800 bold"), # token in front of the question + ("question", "bold"), # question text + ("answer", "fg:#fac800 bold"), # submitted answer text behind the question + ("pointer", "fg:#fac800 bold"), # pointer used in select and checkbox prompts + ("highlighted", "bg:#fac800 fg:#000000 bold"), # pointed-at choice in select and checkbox prompts + ("selected", "fg:#54cc90"), # style for a selected item of a checkbox + ("separator", "fg:#cc5454"), # separator in lists + # ('instruction', ''), # user instructions for select, rawselect, checkbox + ("text", ""), # plain text + # ('disabled', 'fg:#858585 italic') # disabled choices for select and checkbox prompts + ] +) + + +@click.command( + name="pr", + cls=RichCommand, + help=__doc__, +) +@click.argument( + "title", + type=str, + required=True, +) +@click.argument( + "category", + type=click.Choice(list(PR_CATEGORIES.keys()), case_sensitive=False), + required=False, + default=None, +) +@click.option( + "--scope", + "-s", + help="Scope of the PR (only relevant if --title is given). This text will be preprended to the PR title. **Examples**: 'demography' for data work on this field, 'etl.db' if working on specific modules, 'wizard', etc.", + default=None, +) +@click.option( + "--work-branch", + "-w", + "work_branch", + type=str, + default=None, + help="The name of the work branch to create. It is auto-generated based on the title and the category. If --direct is used, this is the PR source branch and defaults to the current branch.", +) +@click.option( + "--base-branch", + "-b", + "base_branch", + type=str, + default="master", + help="Name of the base branch. This is the branch to branch out from and merge back into. If --direct is used, this is the PR target branch.", +) +@click.option( + "--direct", + "-d", + is_flag=True, + help="Directly create a PR from the current branch to the target branch (default: master).", +) +@click.option( + "--private", + "-p", + is_flag=True, + help="By default, staging server site (not admin) will be publicly accessible. Use --private to have it private instead. This does not apply when using --direct mode.", +) +@click.option( + "--no-llm", + "-n", + is_flag=True, + help="We briefly use LLMs to simplify the title and use it in the branch name. Disable this by using -n flag.", +) +def cli( + title: str, + category: Optional[str], + scope: Optional[str], + work_branch: Optional[str], + base_branch: str, + direct: bool, + private: bool, + no_llm: bool, + # base_branch: Optional[str] = None, +) -> None: + # Check that the user has set up a GitHub token. + check_gh_token() + + # Validate title + _validate_title(title) + + # Get category + category = ensure_category(category) + + # Create title + pr_title = PRTitle( + title=title, + category=category, + scope=scope, + ) + + # Initialize repository, get remote branches + repo, remote_branches = init_repo() + + # Get the new branch + work_branch = ensure_work_branch( + repo=repo, + work_branch=work_branch, + direct=direct, + pr_title=pr_title, + remote_branches=remote_branches, + no_llm=no_llm, + ) + + # Check branches main & work make sense! + check_branches_valid(base_branch, work_branch, remote_branches) + + # Auto PR mode: Create a new branch from the base branch. + if not direct: + if private: + if not work_branch.endswith("-private"): + work_branch = f"{work_branch}-private" + branch_out(repo, base_branch, work_branch) + + # Create PR + create_pr(repo, work_branch, base_branch, pr_title) + + +def check_gh_token(): + if not GITHUB_TOKEN: + raise click.ClickException( + """A github token is needed. To create one: +- Go to: https://github.com/settings/tokens +- Click on the dropdown "Generate new token" and select "Generate new token (classic)". +- Give the token a name (e.g., "etl-work"), set an expiration time, and select the scope "repo". +- Click on "Generate token". +- Copy the token and save it in your .env file as GITHUB_TOKEN. +- Run this tool again. +""" + ) + + +def _validate_title(title): + if not bool(re.search(r"\w+", title)): + raise click.ClickException("Invalid title! Use at least one word!") + + +def ensure_category(category: Optional[str]): + """Get category if not provided.""" + if category is None: + # show suggestions + choices = [questionary.Choice(**choice) for choice in PR_CATEGORIES_CHOICES] # type: ignore + category = questionary.select( + message="Please choose a PR category", + choices=choices, + use_shortcuts=True, + style=SHELL_FORM_STYLE, + instruction="(Use shortcuts or arrow keys)", + ).unsafe_ask() + + category = cast(str, category) + + return category + + +class PRTitle: + def __init__(self, title, category, scope): + self.title = title + self.category = category + self.scope = scope + + def __str__(self) -> str: + title_actual = _generate_pr_title(self.title, self.category, self.scope) + if title_actual is None: + raise click.ClickException("Failed to generate PR title.") + return title_actual + + +def init_repo(): + # Initialize a repos object at the root folder of the etl repos. + repo = Repo(BASE_DIR) + # Update the list of remote branches in the local repository. + origin = repo.remote(name="origin") + # NOTE: The option prune=True removes local references to branches that no longer exist on the remote repository. + # Otherwise, this script might raise an error claiming that your proposed branch exists in remote, even if that + # branch was already deleted. + origin.fetch(prune=True) + # List all remote branches. + remote_branches = [ref.name.split("origin/")[-1] for ref in origin.refs if ref.remote_head != "HEAD"] + + return repo, remote_branches + + +def ensure_work_branch(repo, work_branch, direct, pr_title, remote_branches, no_llm): + """Get name of new branch if not provided.""" + # If no name for new branch is given + if work_branch is None: + if not direct: + # Generate name for new branch + work_branch = bake_branch_name(repo, pr_title, no_llm, remote_branches) + else: + # If not explicitly given, the new branch will be the current branch. + work_branch = repo.active_branch.name + if work_branch == "master": + message = "You're currently on 'master' branch. Pass the name of a branch as an argument to create a new branch." + raise click.ClickException(message) + # If a name is given, and not in direct mode + elif (work_branch is not None) & (not direct): + local_branches = [branch.name for branch in repo.branches] + if work_branch in local_branches: + message = ( + f"Branch '{work_branch}' already exists locally." + "Either choose a different name for the new branch to be created, " + "or switch to the new branch and run this tool without specifying a new branch." + ) + raise click.ClickException(message) + return work_branch + + +def check_branches_valid(base_branch, work_branch, remote_branches): + """Ensure the base branch exists in remote (this should always be true for 'master').""" + # Check base branch (main) + if base_branch not in remote_branches: + raise click.ClickException( + f"Base branch '{base_branch}' does not exist in remote. " + "Either push that branch (git push origin base-branch-name) or use 'master' as a base branch. " + "Then run this tool again." + ) + # Check work branch + if work_branch in remote_branches: + raise click.ClickException( + f"New branch '{work_branch}' already exists in remote. " + "Either manually create a pull request from github, or use a different name for the new branch." + ) + + +def branch_out(repo, base_branch, work_branch): + """Branch out from base_branch and create branch 'work_branch'.""" + try: + log.info( + f"Switching to base branch '{base_branch}', creating new branch '{work_branch}' from there, and switching to it." + ) + repo.git.checkout(base_branch) + repo.git.checkout("-b", work_branch) + except GitCommandError as e: + raise click.ClickException(f"Failed to create a new branch from '{base_branch}':\n{e}") + + +def create_pr(repo, work_branch, base_branch, pr_title): + """Create a draft pull request work_branch -> base_branch.""" + pr_title_str = str(pr_title) + + log.info("Creating an empty commit.") + repo.git.commit("--allow-empty", "-m", pr_title_str or f"Start a new staging server for branch '{work_branch}'") + + log.info("Pushing the new branch to remote.") + repo.git.push("origin", work_branch) + + log.info("Creating a draft pull request.") + headers = {"Authorization": f"token {GITHUB_TOKEN}", "Accept": "application/vnd.github.v3+json"} + data = { + "title": pr_title_str or f":construction: Draft PR for branch {work_branch}", + "head": work_branch, + "base": base_branch, + "body": "", + "draft": True, + } + response = requests.post(GITHUB_API_URL, json=data, headers=headers) + if response.status_code == 201: + js = response.json() + log.info(f"Draft pull request created successfully at {js['html_url']}.") + else: + raise click.ClickException(f"Failed to create draft pull request:\n{response.json()}") + + +def _generate_pr_title(title: str, category: str, scope: str | None) -> Optional[str]: + """Generate the PR title. + + title + category + scope -> 'category scope: title' + title + category -> 'category title' + """ + if title is not None: + prefix = "" + # Add emoji for PR mode chosen if applicable + if category in PR_CATEGORIES: + prefix = PR_CATEGORIES[category]["emoji"] + else: + raise ValueError(f"Invalid PR type '{category}'. Choose one of {list(PR_CATEGORIES.keys())}.") + # Add scope + if scope is not None: + if prefix != "": + prefix += " " + prefix += f"{scope}:" + + # Add prefix + title = f"{prefix} {title}" + return title + + +def bake_branch_name(repo, pr_title, no_llm, remote_branches): + # Get user + # git_config = repo.config_reader() + # user = git_config.get_value("user", "name").lower() + + # Get category + category = pr_title.category + + # Get input title (without emoji, scope, etc.) + title = _extract_relevant_title_for_branch_name(pr_title.title, category, not no_llm) + + # Bake complete PR branch name + # name = f"{user}-{category}-{title}" + name = f"{category}-{title}" + + # If branch name collision + # if name in remote_branches: + # log.info("Generating a hash for this branch name to prevent name collisions.") + # name = f"{name}-{user}" + local_branches = [branch.name for branch in repo.branches] + if (name in remote_branches) or (name in local_branches): + log.info("Generating a hash for this branch name to prevent name collisions.") + name = f"{name}-{generate_short_hash()}" + return name + + +def _extract_relevant_title_for_branch_name(text_in: str, category: str, use_llm) -> str: + """ + Process the input string by: + 1. Removing all symbols, keeping only letters and numbers. + 2. Splitting into a list of words/tokens. + 3. Keeping only the first three tokens (or fewer if not available). + 4. Combining the tokens with a '-'. + + Args: + text_in (str): The input text string. + + Returns: + str: The processed string. + """ + if use_llm: + if "OPENAI_API_KEY" in os.environ: + text_in = summarize_title_llm(text_in) + + cleaned_text = re.sub(r"[^a-zA-Z0-9\s]", "", text_in) + + # Split into tokens/words + tokens = cleaned_text.split() + + # Clean if there is word included in category + tokens = [t for t in tokens if t.lower() != category] + + # Keep only the first 3 tokens + tokens = tokens[:3] + + # Combine tokens with '-' + name = "-".join(tokens).lower() + + return name + + +def generate_short_hash() -> str: + """ + Generate a random short hash (6 characters) using SHA256. + + Returns: + str: A 6-character random hash string. + """ + random_data = uuid.uuid4().hex # Generate random data + random_hash = hashlib.sha256(random_data.encode()).hexdigest() # Create hash + return random_hash[:6] # Return the first 6 characters + + +def summarize_title_llm(title) -> str: + sys_prompt = "You are given a title of a pull request. I need a 2-3 keyword summary, separated by a space. These words will be used to create a branch name." + api = OpenAIWrapper() + log.info("Querying GPT!") + response = api.query_gpt_fast(title, sys_prompt, model="gpt-4o-mini") + return response diff --git a/apps/step_update/cli.py b/apps/step_update/cli.py index fa7325a2822..f315be84f6c 100644 --- a/apps/step_update/cli.py +++ b/apps/step_update/cli.py @@ -363,7 +363,8 @@ def update_steps( # Tell user how to automatically create PR short_name = steps[-1].split("/")[-1].split(".")[0] - cmd = f'etl pr update-{short_name} --title ":bar_chart: Update {short_name}"' + # cmd = f'etl pro update-{short_name} --title ":bar_chart: Update {short_name}"' + cmd = f'etl pr "{short_name}" data' log.info(f"Create the PR automatically with:\n {cmd}") def _archive_step(self, step: str) -> None: diff --git a/apps/utils/draft_pull_request.py b/apps/utils/draft_pull_request.py index 2d027c633a4..194a44cfc7b 100644 --- a/apps/utils/draft_pull_request.py +++ b/apps/utils/draft_pull_request.py @@ -96,7 +96,7 @@ def _branch_exists_remotely(new_branch, remote_branches): return False -@click.command(name="pr", cls=RichCommand, help=__doc__) +@click.command(name="pro", cls=RichCommand, help=__doc__) @click.argument( "new-branch", type=str, diff --git a/apps/utils/gpt.py b/apps/utils/gpt.py index 965b19e4472..fa2aa32b84f 100644 --- a/apps/utils/gpt.py +++ b/apps/utils/gpt.py @@ -1,4 +1,5 @@ """Auxiliary classes, functions and variables.""" + from dataclasses import asdict, dataclass, field from typing import Any, Dict, List, Optional, Tuple @@ -217,6 +218,21 @@ def query_gpt( else: raise ValueError("message_content is expected to be a string!") + def query_gpt_fast(self, user_prompt: str, system_prompt: str, model: str = MODEL_DEFAULT) -> str: + """Query Chat GPT to get message content from the chat completion.""" + query = GPTQuery( + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ] + ) + response = self.query_gpt(query=query, model=model) + + if isinstance(response, GPTResponse): + return response.message_content + else: + raise ValueError("message_content is expected to be a string!") + def get_number_tokens(text: str, model_name: str) -> int: """Get number of tokens of text. diff --git a/apps/wizard/app.py b/apps/wizard/app.py index 9ab7a884737..4b6ea6440f2 100644 --- a/apps/wizard/app.py +++ b/apps/wizard/app.py @@ -4,6 +4,7 @@ NOTE: This only works with >1.35 (nightly) version of Streamlit. """ + import streamlit as st from apps.wizard.config import WIZARD_CONFIG diff --git a/apps/wizard/app_pages/anomalist/app.py b/apps/wizard/app_pages/anomalist/app.py index 2a461a1ff57..fb4c92dc65c 100644 --- a/apps/wizard/app_pages/anomalist/app.py +++ b/apps/wizard/app_pages/anomalist/app.py @@ -446,6 +446,9 @@ def show_anomaly_compact(index, df): config = bake_chart_config(variable_id=indicator_id, selected_entities=entities) config["hideAnnotationFieldsInTitle"]["time"] = True config["hideFacetControl"] = False + config["hideShareButton"] = True + config["hideExploreTheDataButton"] = True + # config["isSocialMediaExport"] = False # Actually plot grapher_chart(chart_config=config, owid_env=OWID_ENV) @@ -816,6 +819,7 @@ def _score_table(df: pd.DataFrame) -> pd.DataFrame: # Show controls only if needed if len(items) > items_per_page: pagination.show_controls(mode="bar") - +else: + st.success("Ha! We did not find any no anomalies in the selected datasets! What were the odds of that?") # Reset state set_states({"anomalist_datasets_submitted": False}) diff --git a/apps/wizard/app_pages/anomalist/utils.py b/apps/wizard/app_pages/anomalist/utils.py index 2c724ced7a0..ac7817c8c40 100644 --- a/apps/wizard/app_pages/anomalist/utils.py +++ b/apps/wizard/app_pages/anomalist/utils.py @@ -1,4 +1,5 @@ """Utils for chart revision tool.""" + import time from enum import Enum from typing import Dict, List, Optional, Tuple diff --git a/apps/wizard/app_pages/chart_diff/chart_diff.py b/apps/wizard/app_pages/chart_diff/chart_diff.py index f24a3edbb6c..a1b02055aba 100644 --- a/apps/wizard/app_pages/chart_diff/chart_diff.py +++ b/apps/wizard/app_pages/chart_diff/chart_diff.py @@ -1,5 +1,6 @@ import datetime as dt import difflib +import json import pprint from typing import Any, Dict, List, Optional @@ -10,9 +11,11 @@ from structlog import get_logger from apps.wizard.utils import get_staging_creation_time +from apps.wizard.utils.io import get_all_changed_catalog_paths from etl import grapher_model as gm from etl.config import OWID_ENV from etl.db import read_sql +from etl.git_helpers import get_changed_files, log_time log = get_logger() @@ -200,7 +203,9 @@ def change_types(self) -> list[str]: self._change_types.append("data") if self.modified_checksum["metadataChecksum"].any(): self._change_types.append("metadata") - if self.target_chart and not self.configs_are_equal(): + # NOTE: configs might differ and edited_in_staging is False if the chart had just + # data / metadata changes + if self.edited_in_staging and self.target_chart and not self.configs_are_equal(): self._change_types.append("config") # TODO: Should uncomment this maybe? @@ -353,7 +358,7 @@ def set_conflict_to_resolved(self, session: Session) -> None: def configs_are_equal(self) -> bool: """Compare two chart configs, ignoring version, id and isPublished.""" assert self.target_chart is not None, "Target chart is None!" - return configs_are_equal(self.source_chart.config, self.target_chart.config) + return configs_are_equal(self.source_chart.config, self.target_chart.config, verbose=False) @property def details(self): @@ -535,6 +540,7 @@ def get_diffs_summary_df(self, cache: bool = False, **kwargs) -> pd.DataFrame: return pd.DataFrame(summary) +@log_time def _modified_data_metadata_on_staging( source_session: Session, target_session: Session, chart_ids: Optional[List[int]] = None ) -> pd.DataFrame: @@ -581,6 +587,19 @@ def _modified_data_metadata_on_staging( params["chart_ids"] = tuple(chart_ids) source_df = read_sql(query_source, source_session, params=params) + # no charts, return empty dataframe + if source_df.empty: + return pd.DataFrame(columns=["chartId", "dataEdited", "metadataEdited"]).set_index("chartId") + + # Get all changed files and their catalog paths, including downstream dependencies. + files_changed = get_changed_files() + catalog_paths = get_all_changed_catalog_paths(files_changed) + + # Exclude variables that haven't been changed by updating the files. This is to prevent showing + # spurious changes from lagging behind master. + dataset_paths = source_df.catalogPath.str.split("/").str[:4].str.join("/") + source_df = source_df[dataset_paths.isin(catalog_paths)] + # no charts, return empty dataframe if source_df.empty: return pd.DataFrame(columns=["chartId", "dataEdited", "metadataEdited"]).set_index("chartId") @@ -626,6 +645,7 @@ def _modified_data_metadata_on_staging( return diff +@log_time def _modified_chart_configs_on_staging( source_session: Session, target_session: Session, chart_ids: Optional[List[int]] = None ) -> pd.DataFrame: @@ -636,6 +656,7 @@ def _modified_chart_configs_on_staging( select c.id as chartId, MD5(cc.full) as chartChecksum, + cc.full as chartConfig, c.lastEditedByUserId as chartLastEditedByUserId, c.publishedByUserId as chartPublishedByUserId, c.lastEditedAt as chartLastEditedAt @@ -682,6 +703,20 @@ def _modified_chart_configs_on_staging( diff = source_df.copy() diff["configEdited"] = source_df["chartChecksum"] != target_df["chartChecksum"] + # Go through edited configs and do a more detailed comparison + ix = diff["configEdited"] & target_df["chartChecksum"].notnull() + equal_configs = [] + for chart_id, row in diff.loc[ix].iterrows(): + source_config = json.loads(row["chartConfig"]) + target_config = json.loads(target_df.loc[chart_id, "chartConfig"]) + + # Compare configs + if configs_are_equal(source_config, target_config): + equal_configs.append(chart_id) + + # Exclude configs that have different chartChecksum, but are actually the same (e.g. have just different version) + diff = diff[~diff.index.isin(equal_configs)] + # Add flag 'edited in staging' diff["chartEditedInStaging"] = True diff --git a/apps/wizard/app_pages/chart_diff/chart_diff_show.py b/apps/wizard/app_pages/chart_diff/chart_diff_show.py index b64aa8c44d4..18c8c168f0b 100644 --- a/apps/wizard/app_pages/chart_diff/chart_diff_show.py +++ b/apps/wizard/app_pages/chart_diff/chart_diff_show.py @@ -3,6 +3,7 @@ If you want to learn more about it, start from its `show` method. """ + import difflib import json import os diff --git a/apps/wizard/app_pages/dashboard.py b/apps/wizard/app_pages/dashboard.py index 50fe6f67403..6c50f623c7d 100644 --- a/apps/wizard/app_pages/dashboard.py +++ b/apps/wizard/app_pages/dashboard.py @@ -1,6 +1,5 @@ -"""Create a dashboard with the main information about ETL steps, and the possibility to update them. +"""Create a dashboard with the main information about ETL steps, and the possibility to update them.""" -""" import subprocess from datetime import datetime diff --git a/apps/wizard/app_pages/dataset_explorer.py b/apps/wizard/app_pages/dataset_explorer.py index a297714ad3a..93af44571da 100644 --- a/apps/wizard/app_pages/dataset_explorer.py +++ b/apps/wizard/app_pages/dataset_explorer.py @@ -3,6 +3,7 @@ - [ ] See its dependencies - [ ] Preview its metadata """ + import tempfile from typing import Any, Dict, List, cast diff --git a/apps/wizard/app_pages/expert/app.py b/apps/wizard/app_pages/expert/app.py index 0b2e9de155a..929a40e5439 100644 --- a/apps/wizard/app_pages/expert/app.py +++ b/apps/wizard/app_pages/expert/app.py @@ -3,6 +3,7 @@ references: - https://docs.streamlit.io/knowledge-base/tutorials/build-conversational-apps#build-a-chatgpt-like-app """ + from typing import Any, Dict, cast import streamlit as st diff --git a/apps/wizard/app_pages/expert/prompts.py b/apps/wizard/app_pages/expert/prompts.py index 364deecede3..823b6f8aa9e 100644 --- a/apps/wizard/app_pages/expert/prompts.py +++ b/apps/wizard/app_pages/expert/prompts.py @@ -1,4 +1,5 @@ -""""Get prompts for GPT-interaction.""" +""" "Get prompts for GPT-interaction.""" + import glob from typing import List diff --git a/apps/wizard/app_pages/explorer_edit.py b/apps/wizard/app_pages/explorer_edit.py index 6d41db75d80..99073180608 100644 --- a/apps/wizard/app_pages/explorer_edit.py +++ b/apps/wizard/app_pages/explorer_edit.py @@ -1,4 +1,5 @@ """Helper tool to create map brackets for all indicators in an indicator-based explorer.""" + from io import StringIO import streamlit as st diff --git a/apps/wizard/app_pages/fasttrack/app.py b/apps/wizard/app_pages/fasttrack/app.py index ef7dcbb0232..d561191ef83 100644 --- a/apps/wizard/app_pages/fasttrack/app.py +++ b/apps/wizard/app_pages/fasttrack/app.py @@ -2,6 +2,7 @@ TODO: Fully decouple front-end from backend (see fasttrack.utils for back-end, i.e. should not have streamlit calls). """ + from pathlib import Path import pandas as pd diff --git a/apps/wizard/app_pages/fasttrack/fast_import.py b/apps/wizard/app_pages/fasttrack/fast_import.py index a520a1b9bde..f7edb2fe845 100644 --- a/apps/wizard/app_pages/fasttrack/fast_import.py +++ b/apps/wizard/app_pages/fasttrack/fast_import.py @@ -1,4 +1,5 @@ """Definition of FasttrackImport object (mainly backend).""" + import datetime as dt import difflib import html diff --git a/apps/wizard/app_pages/fasttrack/load.py b/apps/wizard/app_pages/fasttrack/load.py index 7490f088b2b..ec18f7d9d9a 100644 --- a/apps/wizard/app_pages/fasttrack/load.py +++ b/apps/wizard/app_pages/fasttrack/load.py @@ -2,6 +2,7 @@ Relies on Streamlit to print messages. """ + import concurrent.futures import datetime as dt import json @@ -294,7 +295,7 @@ def _parse_sources(sources_meta_df: pd.DataFrame) -> Optional[Source]: source = sources[0] if pd.isnull(source.get("date_accessed")): - source.pop("date_accessed") + source.pop("date_accessed", None) if pd.isnull(source.get("publication_year")): source.pop("publication_year") diff --git a/apps/wizard/app_pages/fasttrack/process.py b/apps/wizard/app_pages/fasttrack/process.py index 8f78ac1f5ac..df0bbacb306 100644 --- a/apps/wizard/app_pages/fasttrack/process.py +++ b/apps/wizard/app_pages/fasttrack/process.py @@ -2,6 +2,7 @@ Relies on Streamlit. """ + import json from typing import Dict, List, Tuple @@ -151,7 +152,7 @@ def _convert_percentages(data: pd.DataFrame, variables_meta_dict: Dict[str, Vari """Convert percentages to numbers.""" for col in data.columns: if getattr(variables_meta_dict.get(col, {}), "unit", "") == "%": - data[col] = data[col].str.replace("%", "").astype(float) + data[col] = data[col].astype(str).str.replace("%", "").astype(float) return data diff --git a/apps/wizard/app_pages/fasttrack/utils.py b/apps/wizard/app_pages/fasttrack/utils.py index 5054a659783..41d49cb969e 100644 --- a/apps/wizard/app_pages/fasttrack/utils.py +++ b/apps/wizard/app_pages/fasttrack/utils.py @@ -1,4 +1,5 @@ """Utils.""" + import os from typing import Any, Dict, Optional diff --git a/apps/wizard/app_pages/harmonizer.py b/apps/wizard/app_pages/harmonizer.py index 3c4285fa425..eea2c69bb57 100644 --- a/apps/wizard/app_pages/harmonizer.py +++ b/apps/wizard/app_pages/harmonizer.py @@ -1,4 +1,5 @@ """Harmonize entities.""" + import json from operator import itemgetter from pathlib import Path diff --git a/apps/wizard/app_pages/indicator_upgrade/app.py b/apps/wizard/app_pages/indicator_upgrade/app.py index 8b0ee1391a8..91f4423974e 100644 --- a/apps/wizard/app_pages/indicator_upgrade/app.py +++ b/apps/wizard/app_pages/indicator_upgrade/app.py @@ -22,6 +22,7 @@ - Set to True: When the user clicks on "Finish (3/3)" in the third form. - Set to False: """ + import streamlit as st from structlog import get_logger diff --git a/apps/wizard/app_pages/indicator_upgrade/charts_update.py b/apps/wizard/app_pages/indicator_upgrade/charts_update.py index 7707ea540d4..420b9a25799 100644 --- a/apps/wizard/app_pages/indicator_upgrade/charts_update.py +++ b/apps/wizard/app_pages/indicator_upgrade/charts_update.py @@ -1,4 +1,5 @@ """Handle submission of chart updates.""" + from http.client import RemoteDisconnected from typing import Dict, List from urllib.error import URLError diff --git a/apps/wizard/app_pages/indicator_upgrade/dataset_selection.py b/apps/wizard/app_pages/indicator_upgrade/dataset_selection.py index dd43dba0362..ec53e3e93cd 100644 --- a/apps/wizard/app_pages/indicator_upgrade/dataset_selection.py +++ b/apps/wizard/app_pages/indicator_upgrade/dataset_selection.py @@ -1,4 +1,5 @@ """Search configuration.""" + from typing import Any, Dict import pandas as pd diff --git a/apps/wizard/app_pages/indicator_upgrade/explore_mode.py b/apps/wizard/app_pages/indicator_upgrade/explore_mode.py index 5e52896dec3..9fe042c8fba 100644 --- a/apps/wizard/app_pages/indicator_upgrade/explore_mode.py +++ b/apps/wizard/app_pages/indicator_upgrade/explore_mode.py @@ -2,6 +2,7 @@ This is currently shown in the indicator upgrader, but might be moved to chart-diff in the future. """ + from dataclasses import dataclass, field from typing import Dict, Optional, Tuple, cast diff --git a/apps/wizard/app_pages/indicator_upgrade/utils.py b/apps/wizard/app_pages/indicator_upgrade/utils.py index f432ca6cffc..01f72dc487c 100644 --- a/apps/wizard/app_pages/indicator_upgrade/utils.py +++ b/apps/wizard/app_pages/indicator_upgrade/utils.py @@ -1,4 +1,5 @@ """Utils for chart revision tool.""" + from typing import Dict, Tuple, cast import pandas as pd diff --git a/apps/wizard/app_pages/map_brackets.py b/apps/wizard/app_pages/map_brackets.py index 9975ebc55ba..78763bd229e 100644 --- a/apps/wizard/app_pages/map_brackets.py +++ b/apps/wizard/app_pages/map_brackets.py @@ -1,6 +1,4 @@ -"""Helper tool to create map brackets for all indicators in an indicator-based explorer. - -""" +"""Helper tool to create map brackets for all indicators in an indicator-based explorer.""" import json from pathlib import Path diff --git a/apps/wizard/app_pages/metagpt.py b/apps/wizard/app_pages/metagpt.py index 68bee27aab9..49069c18b1f 100644 --- a/apps/wizard/app_pages/metagpt.py +++ b/apps/wizard/app_pages/metagpt.py @@ -2,6 +2,7 @@ Note that it still relies on apps/metagpt. """ + import os import tempfile from pathlib import Path diff --git a/apps/wizard/app_pages/metaplay.py b/apps/wizard/app_pages/metaplay.py index c045a0aa95b..086c79140ba 100644 --- a/apps/wizard/app_pages/metaplay.py +++ b/apps/wizard/app_pages/metaplay.py @@ -2,6 +2,7 @@ streamlit run app.py """ + import webbrowser from pathlib import Path diff --git a/apps/wizard/app_pages/news.py b/apps/wizard/app_pages/news.py index e7d9d6bc289..ce6af552949 100644 --- a/apps/wizard/app_pages/news.py +++ b/apps/wizard/app_pages/news.py @@ -1,4 +1,5 @@ """Display news from ETL.""" + from datetime import datetime import streamlit as st diff --git a/apps/wizard/app_pages/owidle.py b/apps/wizard/app_pages/owidle.py index 6afe082f53d..513d0aca78d 100644 --- a/apps/wizard/app_pages/owidle.py +++ b/apps/wizard/app_pages/owidle.py @@ -1,4 +1,5 @@ """Game owidle.""" + import datetime as dt import math from itertools import product diff --git a/apps/wizard/cli.py b/apps/wizard/cli.py index aedf01a6f21..189473220e4 100644 --- a/apps/wizard/cli.py +++ b/apps/wizard/cli.py @@ -4,6 +4,7 @@ python cli.py """ + import logging import sys from typing import Iterable diff --git a/apps/wizard/etl_steps/express.py b/apps/wizard/etl_steps/express.py index a14429a87f8..d1f4e6ba2a5 100644 --- a/apps/wizard/etl_steps/express.py +++ b/apps/wizard/etl_steps/express.py @@ -1,4 +1,5 @@ """Garden phase.""" + from datetime import date, datetime, timedelta from pathlib import Path from typing import Any, Dict, List diff --git a/apps/wizard/etl_steps/garden.py b/apps/wizard/etl_steps/garden.py index a7194639b2b..b2bee29c309 100644 --- a/apps/wizard/etl_steps/garden.py +++ b/apps/wizard/etl_steps/garden.py @@ -1,4 +1,5 @@ """Garden phase.""" + import os from pathlib import Path from typing import List, cast @@ -452,9 +453,9 @@ def export_metadata() -> None: f"Use the generated notebook `{notebook_path.relative_to(BASE_DIR)}` to examine the dataset output interactively." ) # B/ Generate metadata - st.session_state[ - "garden.dataset_path" - ] = f"data/garden/{form.namespace}/{form.version}/{form.short_name}" + st.session_state["garden.dataset_path"] = ( + f"data/garden/{form.namespace}/{form.version}/{form.short_name}" + ) st.markdown("#### Generate metadata") st.markdown(f"Generate metadata file `{form.short_name}.meta.yml` from your dataset with:") st.button( diff --git a/apps/wizard/etl_steps/grapher.py b/apps/wizard/etl_steps/grapher.py index 85e1aa1b925..0b2618d520c 100644 --- a/apps/wizard/etl_steps/grapher.py +++ b/apps/wizard/etl_steps/grapher.py @@ -1,4 +1,5 @@ """Grapher phase.""" + from pathlib import Path from typing import cast diff --git a/apps/wizard/etl_steps/meadow.py b/apps/wizard/etl_steps/meadow.py index eab35019502..f7c452d0964 100644 --- a/apps/wizard/etl_steps/meadow.py +++ b/apps/wizard/etl_steps/meadow.py @@ -1,4 +1,5 @@ """Meadow phase.""" + import os from pathlib import Path from typing import cast diff --git a/apps/wizard/etl_steps/snapshot.py b/apps/wizard/etl_steps/snapshot.py index 8cc5b71c90d..06b1b53c06e 100644 --- a/apps/wizard/etl_steps/snapshot.py +++ b/apps/wizard/etl_steps/snapshot.py @@ -1,4 +1,5 @@ """Snapshot phase.""" + import os import subprocess import traceback diff --git a/apps/wizard/home.py b/apps/wizard/home.py index 06c51bacb58..bde6d576242 100644 --- a/apps/wizard/home.py +++ b/apps/wizard/home.py @@ -1,4 +1,5 @@ """Home page of wizard.""" + from copy import deepcopy from typing import Any, Dict, List, Optional diff --git a/apps/wizard/scripts/news.py b/apps/wizard/scripts/news.py index 0bdd5b3e610..40cc88716ba 100644 --- a/apps/wizard/scripts/news.py +++ b/apps/wizard/scripts/news.py @@ -1,4 +1,5 @@ """Methods to update Wizard DB to have the necessary PR data.""" + from typing import Tuple import requests diff --git a/apps/wizard/utils/__init__.py b/apps/wizard/utils/__init__.py index 89ad8753306..82cf0de820b 100644 --- a/apps/wizard/utils/__init__.py +++ b/apps/wizard/utils/__init__.py @@ -9,6 +9,7 @@ Also, can imagine apps/wizard/ being renamed to just wizard/, and stuff other than wizard should be either (i) deleted or (ii) migrated elsewhere in etl/. """ + import argparse import ast import datetime as dt diff --git a/apps/wizard/utils/chart_config.py b/apps/wizard/utils/chart_config.py index 82dbd0b2890..344487d7a93 100644 --- a/apps/wizard/utils/chart_config.py +++ b/apps/wizard/utils/chart_config.py @@ -1,4 +1,5 @@ """Tools to generate chart configs.""" + from copy import deepcopy from typing import Any, Dict, List, Optional diff --git a/apps/wizard/utils/db.py b/apps/wizard/utils/db.py index 7abad13ecb8..7de9d3c6873 100644 --- a/apps/wizard/utils/db.py +++ b/apps/wizard/utils/db.py @@ -5,6 +5,7 @@ Some of the tools here rely on a local temporary sqlite database. This database is a custom and temporary database used to store data in a server. Not intended for production use. """ + import datetime as dt import hashlib import os diff --git a/apps/wizard/utils/io.py b/apps/wizard/utils/io.py index c131fe02b30..e16a0596f3f 100644 --- a/apps/wizard/utils/io.py +++ b/apps/wizard/utils/io.py @@ -2,6 +2,7 @@ Together with utils.db and utils.cached, it might need some rethinking on where it goes. """ + from pathlib import Path from typing import Dict, List, Optional @@ -14,7 +15,8 @@ from apps.wizard.utils.cached import get_datasets_from_version_tracker from etl.git_helpers import get_changed_files from etl.grapher_io import get_all_datasets -from etl.paths import BASE_DIR, STEP_DIR +from etl.paths import BASE_DIR, SNAPSHOTS_DIR, STEP_DIR +from etl.steps import filter_to_subgraph, load_dag # Initialize logger. log = get_logger() @@ -61,9 +63,8 @@ def get_steps_df(archived: bool = True): ######################################################################################################################## -def get_changed_grapher_steps(files_changed: Dict[str, Dict[str, str]]) -> List[str]: - """Get list of new grapher steps with their corresponding old steps.""" - grapher_steps = [] +def get_changed_steps(files_changed: Dict[str, Dict[str, str]]) -> List[str]: + changed_steps = [] for file_path, file_status in files_changed.items(): # File status can be: D (deleted), A (added), M (modified). # NOTE: In principle, we could select only "A" files. But it is possible that the user adds a new grapher step, and then commits changes to it, in which case (I think) the status would be "M". @@ -73,15 +74,26 @@ def get_changed_grapher_steps(files_changed: Dict[str, Dict[str, str]]) -> List[ # Skip deleted files. continue - # Identify grapher data steps, and ignore the rest. - if file_path.startswith(STEP_DIR.relative_to(BASE_DIR).as_posix()) and file_path.endswith(".py"): - parts = Path(file_path).with_suffix("").as_posix().split("/") - if len(parts) == 4 and parts[-4] == "grapher": - grapher_steps.append(file_path) + # Identify potential recipes for data steps + if file_path.startswith( + (STEP_DIR.relative_to(BASE_DIR).as_posix(), SNAPSHOTS_DIR.relative_to(BASE_DIR).as_posix()) + ): + changed_steps.append(file_path) else: continue - return grapher_steps + return changed_steps + + +def get_changed_grapher_steps(files_changed: Dict[str, Dict[str, str]]) -> List[str]: + """Get list of new grapher steps with their corresponding old steps.""" + steps = [] + for step_path in get_changed_steps(files_changed): + if step_path.endswith(".py"): + parts = Path(step_path).with_suffix("").as_posix().split("/") + if len(parts) >= 4 and parts[-4] == "grapher": + steps.append(step_path) + return steps def get_new_grapher_datasets_and_their_previous_versions(session: Session) -> Dict[int, Optional[int]]: @@ -137,3 +149,37 @@ def get_new_grapher_datasets_and_their_previous_versions(session: Session) -> Di new_datasets[ds_id] = previous_dataset return new_datasets + + +def get_all_changed_catalog_paths(files_changed: Dict[str, Dict[str, str]]) -> List[str]: + """Get all changed steps and their downstream dependencies.""" + dataset_catalog_paths = [] + + # Get catalog paths of all datasets with changed files. + for step_path in get_changed_steps(files_changed): + abs_step_path = BASE_DIR / Path(step_path) + try: + # TODO: use StepPath from https://github.com/owid/etl/pull/3165 to refactor this + if step_path.startswith("snapshots/"): + ds_path = abs_step_path.relative_to(SNAPSHOTS_DIR).with_suffix("").with_suffix("").as_posix() + else: + ds_path = abs_step_path.relative_to(STEP_DIR / "data").with_suffix("").with_suffix("").as_posix() + dataset_catalog_paths.append(ds_path) + except ValueError: + continue + + # NOTE: + # This is OK, as it filters down the DAG a little bit. But using VersionTracker.steps_df would be much more precise. You could do: + # steps_df[(steps_df["step"].isin([...])]["all_active_usages"] + # And that would give you only the steps that are affected by the changed files. That would be ultimately what we need. But I + # understand that loading steps_df is very slow. + + # Add all downstream dependencies of those datasets. + DAG = load_dag() + dag_steps = list(filter_to_subgraph(DAG, dataset_catalog_paths, downstream=True).keys()) + + # From data://... extract catalogPath + # TODO: use StepPath from https://github.com/owid/etl/pull/3165 to refactor this + catalog_paths = [step.split("://")[1] for step in dag_steps if step.startswith("data://")] + + return catalog_paths diff --git a/dag/archive/artificial_intelligence.yml b/dag/archive/artificial_intelligence.yml index f15060fc02a..a69ed71a8d4 100644 --- a/dag/archive/artificial_intelligence.yml +++ b/dag/archive/artificial_intelligence.yml @@ -1,6 +1,7 @@ steps: ############################################################################################################## # EPOCH archive (monthly updates) + # Artificial Intelligence (EPOCH) data://meadow/artificial_intelligence/latest/epoch: - snapshot://artificial_intelligence/latest/epoch.csv @@ -236,6 +237,53 @@ steps: data://grapher/artificial_intelligence/2024-10-01/epoch_compute_intensive_domain: - data://garden/artificial_intelligence/2024-10-01/epoch_compute_intensive_domain + # Main EPOCH dataset + data://meadow/artificial_intelligence/2024-11-03/epoch: + - snapshot://artificial_intelligence/2024-11-03/epoch.csv + data://garden/artificial_intelligence/2024-11-03/epoch: + - data://meadow/artificial_intelligence/2024-11-03/epoch + data://grapher/artificial_intelligence/2024-11-03/epoch: + - data://garden/artificial_intelligence/2024-11-03/epoch + + # Main EPOCH dataset regression lines + data://garden/artificial_intelligence/2024-11-03/epoch_regressions: + - data://garden/artificial_intelligence/2024-11-03/epoch + data://grapher/artificial_intelligence/2024-11-03/epoch_regressions: + - data://garden/artificial_intelligence/2024-11-03/epoch_regressions + + # EPOCH aggregates by domain + data://garden/artificial_intelligence/2024-11-03/epoch_aggregates_domain: + - data://meadow/artificial_intelligence/2024-11-03/epoch + data://grapher/artificial_intelligence/2024-11-03/epoch_aggregates_domain: + - data://garden/artificial_intelligence/2024-11-03/epoch_aggregates_domain + + # EPOCH aggregates by researcher affiliaiton + data://garden/artificial_intelligence/2024-11-03/epoch_aggregates_affiliation: + - data://garden/artificial_intelligence/2024-11-03/epoch + data://grapher/artificial_intelligence/2024-11-03/epoch_aggregates_affiliation: + - data://garden/artificial_intelligence/2024-11-03/epoch_aggregates_affiliation + + # EPOCH dataset on Compute Intensive AI + data://meadow/artificial_intelligence/2024-11-03/epoch_compute_intensive: + - snapshot://artificial_intelligence/2024-11-03/epoch_compute_intensive.csv + data://garden/artificial_intelligence/2024-11-03/epoch_compute_intensive: + - data://meadow/artificial_intelligence/2024-11-03/epoch_compute_intensive + + + # EPOCH dataset on Compute Intensive AI, aggregates by country + data://garden/artificial_intelligence/2024-11-03/epoch_compute_intensive_countries: + - data://meadow/artificial_intelligence/2024-11-03/epoch_compute_intensive + data://grapher/artificial_intelligence/2024-11-03/epoch_compute_intensive_countries: + - data://garden/artificial_intelligence/2024-11-03/epoch_compute_intensive_countries + + # EPOCH dataset on Compute Intensive AI, aggregates by domain + data://garden/artificial_intelligence/2024-11-03/epoch_compute_intensive_domain: + - data://meadow/artificial_intelligence/2024-11-03/epoch_compute_intensive + data://grapher/artificial_intelligence/2024-11-03/epoch_compute_intensive_domain: + - data://garden/artificial_intelligence/2024-11-03/epoch_compute_intensive_domain + + + ############################################################################################################## # AI Incidents diff --git a/dag/archive/demography.yml b/dag/archive/demography.yml index 185500becad..7cbb7a7ee38 100644 --- a/dag/archive/demography.yml +++ b/dag/archive/demography.yml @@ -52,9 +52,77 @@ steps: data-private://grapher/un/2024-07-11/un_wpp_full: - data-private://garden/un/2024-07-11/un_wpp - # Population density + # Population density data://garden/demography/2023-06-12/population_density: - data://garden/demography/2023-03-31/population - data://garden/faostat/2024-03-14/faostat_rl data://grapher/demography/2023-06-12/population_density: - data://garden/demography/2023-06-12/population_density + + # HMD + data://meadow/hmd/2022-12-07/life_tables: + - snapshot://hmd/2022-12-07/hmd.zip + data://garden/hmd/2022-12-07/life_tables: + - data://meadow/hmd/2022-12-07/life_tables + + # Survivorship ages (HMD-derived) + data://garden/demography/2023-09-27/survivor_percentiles: + - data://garden/hmd/2023-09-19/hmd + data://grapher/demography/2023-09-27/survivor_percentiles: + - data://garden/demography/2023-09-27/survivor_percentiles + + # Phi-gender life expectancy inequality + data://garden/demography/2023-10-03/phi_gender_le: + - data://garden/demography/2023-10-03/life_tables + data://grapher/demography/2023-10-03/phi_gender_le: + - data://garden/demography/2023-10-03/phi_gender_le + + # Broken limits of Life Expectancy + data://garden/demography/2023-10-20/broken_limits_le: + - data://garden/demography/2023-10-03/life_tables + - data://garden/hmd/2023-09-19/hmd + data://grapher/demography/2023-10-20/broken_limits_le: + - data://garden/demography/2023-10-20/broken_limits_le + + # Gini Life Expectancy Inequality + data://garden/demography/2023-10-04/gini_le: + - data://garden/demography/2023-10-03/life_tables + data://grapher/demography/2023-10-04/gini_le: + - data://garden/demography/2023-10-04/gini_le + + # HMD + data://meadow/hmd/2023-09-19/hmd: + - snapshot://hmd/2023-09-18/hmd.zip + data://garden/hmd/2023-09-19/hmd: + - data://meadow/hmd/2023-09-19/hmd + data://grapher/hmd/2023-09-19/hmd: + - data://garden/hmd/2023-09-19/hmd + # UN WPP Life Tables + data://meadow/un/2023-10-02/un_wpp_lt: + - snapshot://un/2023-10-02/un_wpp_lt_all.zip + - snapshot://un/2023-10-02/un_wpp_lt_f.zip + - snapshot://un/2023-10-02/un_wpp_lt_m.zip + data://garden/un/2023-10-02/un_wpp_lt: + - data://meadow/un/2023-10-02/un_wpp_lt + # UN WPP + HMD Life Tables + data://garden/demography/2023-10-03/life_tables: + - data://garden/hmd/2023-09-19/hmd + - data://garden/un/2023-10-02/un_wpp_lt + data://grapher/demography/2023-10-04/life_tables: + - data://garden/demography/2023-10-03/life_tables + # OMM: Life Expectancy + data://garden/demography/2023-10-09/life_expectancy: + - data://garden/demography/2023-10-03/life_tables + - data://garden/demography/2023-10-10/zijdeman_et_al_2015 + - data://garden/demography/2023-10-10/riley_2005 + - data://garden/un/2022-07-11/un_wpp + data://grapher/demography/2023-10-10/life_expectancy: + - data://garden/demography/2023-10-09/life_expectancy + + ## Population by education group (Wittgenstein centre) + data://meadow/demography/2023-04-07/population_education_wittgenstein: + - snapshot://demography/2023-04-07/population_education_wittgenstein.csv + data://garden/demography/2023-04-07/population_education_wittgenstein: + - data://meadow/demography/2023-04-07/population_education_wittgenstein + data://grapher/demography/2023-04-07/population_education_wittgenstein: + - data://garden/demography/2023-04-07/population_education_wittgenstein diff --git a/dag/archive/education.yml b/dag/archive/education.yml index 7fb0ac85e2b..1359bbefef6 100644 --- a/dag/archive/education.yml +++ b/dag/archive/education.yml @@ -9,4 +9,14 @@ steps: - data://garden/education/2018-04-18/literacy_rates data://grapher/wb/2023-07-10/education: - - data://garden/wb/2023-07-10/education \ No newline at end of file + - data://garden/wb/2023-07-10/education + + # Wittgenstein Center Population and Human Capital Projections - Educational Attainment + data://meadow/education/2023-12-15/wittgenstein_center: + - snapshot://education/2023-12-15/wittgenstein_center_data.csv + - snapshot://education/2023-12-15/wittgenstein_center_dictionary.csv + data://garden/education/2023-12-15/wittgenstein_center: + - data://meadow/education/2023-12-15/wittgenstein_center + - data://garden/education/2023-08-14/oecd_education + data://grapher/education/2023-12-15/wittgenstein_center: + - data://garden/education/2023-12-15/wittgenstein_center diff --git a/dag/archive/fasttrack.yml b/dag/archive/fasttrack.yml index 9f49691a2bd..af669e2acdc 100644 --- a/dag/archive/fasttrack.yml +++ b/dag/archive/fasttrack.yml @@ -1,5 +1,5 @@ steps: - data://grapher/fasttrack/2023-06-19/world_population_comparison: - - snapshot://fasttrack/2023-06-19/world_population_comparison.csv data://grapher/fasttrack/latest/democracy_lexical_index: - snapshot://fasttrack/latest/democracy_lexical_index.csv + data://grapher/fasttrack/latest/mineral_prices_usgs: + - snapshot://fasttrack/latest/mineral_prices_usgs.csv diff --git a/dag/archive/health.yml b/dag/archive/health.yml index 9d9813fc384..2722545fa83 100644 --- a/dag/archive/health.yml +++ b/dag/archive/health.yml @@ -38,7 +38,7 @@ steps: - data://meadow/oecd/2023-08-11/road_accidents - data://garden/oecd/2018-03-11/road_deaths_and_injuries -# GBD 2019 + # GBD 2019 # IHME GBD Leading cause of deaths - to archive data://meadow/ihme_gbd/2023-10-04/cause_hierarchy: @@ -49,7 +49,7 @@ steps: data://grapher/ihme_gbd/2023-10-04/leading_causes_child_deaths: - data://garden/ihme_gbd/2023-10-04/leading_causes_child_deaths -# IHME Global Burden of Disease - Prevalence and incidence + # IHME Global Burden of Disease - Prevalence and incidence data://meadow/ihme_gbd/2019/gbd_prevalence: - walden://ihme_gbd/2019/gbd_prevalence data://garden/ihme_gbd/2019/gbd_prevalence: @@ -118,4 +118,12 @@ steps: data://garden/who/2023-08-04/icd_codes: - data://meadow/who/2023-08-04/icd_codes data://grapher/who/2023-08-04/icd_codes: - - data://garden/who/2023-08-04/icd_codes \ No newline at end of file + - data://garden/who/2023-08-04/icd_codes + + # MICROBE - removing some unneeded datasets from grapher + data-private://grapher/antibiotics/2024-12-04/microbe_total_pathogens: + - data-private://garden/antibiotics/2024-12-04/microbe_total_pathogens + data-private://grapher/antibiotics/2024-12-02/total_pathogen_bloodstream: + - data-private://garden/antibiotics/2024-12-02/total_pathogen_bloodstream + data-private://grapher/antibiotics/2024-11-20/pathogen_bloodstream: + - data-private://garden/antibiotics/2024-11-20/pathogen_bloodstream diff --git a/dag/archive/poverty_inequality.yml b/dag/archive/poverty_inequality.yml index e2229fd573f..8ca39d6197c 100644 --- a/dag/archive/poverty_inequality.yml +++ b/dag/archive/poverty_inequality.yml @@ -43,3 +43,12 @@ steps: - data://meadow/ophi/2023-07-05/multidimensional_poverty_index data://grapher/ophi/2023-07-05/multidimensional_poverty_index: - data://garden/ophi/2023-07-05/multidimensional_poverty_index + + # Poverty projections from the World Bank + data://meadow/wb/2024-06-26/poverty_projections: + - snapshot://wb/2024-06-26/poverty_projections_number_global.csv + - snapshot://wb/2024-06-26/poverty_projections_share_regions.csv + data://garden/wb/2024-06-26/poverty_projections: + - data://meadow/wb/2024-06-26/poverty_projections + data://grapher/wb/2024-06-26/poverty_projections: + - data://garden/wb/2024-06-26/poverty_projections diff --git a/dag/artificial_intelligence.yml b/dag/artificial_intelligence.yml index a18f9550a83..868a40ad852 100644 --- a/dag/artificial_intelligence.yml +++ b/dag/artificial_intelligence.yml @@ -1,49 +1,49 @@ steps: ########### UPDATED MONTHLY ############################################################################# # Main EPOCH dataset - data://meadow/artificial_intelligence/2024-11-03/epoch: - - snapshot://artificial_intelligence/2024-11-03/epoch.csv - data://garden/artificial_intelligence/2024-11-03/epoch: - - data://meadow/artificial_intelligence/2024-11-03/epoch - data://grapher/artificial_intelligence/2024-11-03/epoch: - - data://garden/artificial_intelligence/2024-11-03/epoch + data://meadow/artificial_intelligence/2024-12-05/epoch: + - snapshot://artificial_intelligence/2024-12-05/epoch.csv + data://garden/artificial_intelligence/2024-12-05/epoch: + - data://meadow/artificial_intelligence/2024-12-05/epoch + data://grapher/artificial_intelligence/2024-12-05/epoch: + - data://garden/artificial_intelligence/2024-12-05/epoch # Main EPOCH dataset regression lines - data://garden/artificial_intelligence/2024-11-03/epoch_regressions: - - data://garden/artificial_intelligence/2024-11-03/epoch - data://grapher/artificial_intelligence/2024-11-03/epoch_regressions: - - data://garden/artificial_intelligence/2024-11-03/epoch_regressions + data://garden/artificial_intelligence/2024-12-05/epoch_regressions: + - data://garden/artificial_intelligence/2024-12-05/epoch + data://grapher/artificial_intelligence/2024-12-05/epoch_regressions: + - data://garden/artificial_intelligence/2024-12-05/epoch_regressions # EPOCH aggregates by domain - data://garden/artificial_intelligence/2024-11-03/epoch_aggregates_domain: - - data://meadow/artificial_intelligence/2024-11-03/epoch - data://grapher/artificial_intelligence/2024-11-03/epoch_aggregates_domain: - - data://garden/artificial_intelligence/2024-11-03/epoch_aggregates_domain + data://garden/artificial_intelligence/2024-12-05/epoch_aggregates_domain: + - data://meadow/artificial_intelligence/2024-12-05/epoch + data://grapher/artificial_intelligence/2024-12-05/epoch_aggregates_domain: + - data://garden/artificial_intelligence/2024-12-05/epoch_aggregates_domain # EPOCH aggregates by researcher affiliaiton - data://garden/artificial_intelligence/2024-11-03/epoch_aggregates_affiliation: - - data://garden/artificial_intelligence/2024-11-03/epoch - data://grapher/artificial_intelligence/2024-11-03/epoch_aggregates_affiliation: - - data://garden/artificial_intelligence/2024-11-03/epoch_aggregates_affiliation + data://garden/artificial_intelligence/2024-12-05/epoch_aggregates_affiliation: + - data://garden/artificial_intelligence/2024-12-05/epoch + data://grapher/artificial_intelligence/2024-12-05/epoch_aggregates_affiliation: + - data://garden/artificial_intelligence/2024-12-05/epoch_aggregates_affiliation # EPOCH dataset on Compute Intensive AI - data://meadow/artificial_intelligence/2024-11-03/epoch_compute_intensive: - - snapshot://artificial_intelligence/2024-11-03/epoch_compute_intensive.csv - data://garden/artificial_intelligence/2024-11-03/epoch_compute_intensive: - - data://meadow/artificial_intelligence/2024-11-03/epoch_compute_intensive + data://meadow/artificial_intelligence/2024-12-05/epoch_compute_intensive: + - snapshot://artificial_intelligence/2024-12-05/epoch_compute_intensive.csv + data://garden/artificial_intelligence/2024-12-05/epoch_compute_intensive: + - data://meadow/artificial_intelligence/2024-12-05/epoch_compute_intensive # EPOCH dataset on Compute Intensive AI, aggregates by country - data://garden/artificial_intelligence/2024-11-03/epoch_compute_intensive_countries: - - data://meadow/artificial_intelligence/2024-11-03/epoch_compute_intensive - data://grapher/artificial_intelligence/2024-11-03/epoch_compute_intensive_countries: - - data://garden/artificial_intelligence/2024-11-03/epoch_compute_intensive_countries + data://garden/artificial_intelligence/2024-12-05/epoch_compute_intensive_countries: + - data://meadow/artificial_intelligence/2024-12-05/epoch_compute_intensive + data://grapher/artificial_intelligence/2024-12-05/epoch_compute_intensive_countries: + - data://garden/artificial_intelligence/2024-12-05/epoch_compute_intensive_countries # EPOCH dataset on Compute Intensive AI, aggregates by domain - data://garden/artificial_intelligence/2024-11-03/epoch_compute_intensive_domain: - - data://meadow/artificial_intelligence/2024-11-03/epoch_compute_intensive - data://grapher/artificial_intelligence/2024-11-03/epoch_compute_intensive_domain: - - data://garden/artificial_intelligence/2024-11-03/epoch_compute_intensive_domain + data://garden/artificial_intelligence/2024-12-05/epoch_compute_intensive_domain: + - data://meadow/artificial_intelligence/2024-12-05/epoch_compute_intensive + data://grapher/artificial_intelligence/2024-12-05/epoch_compute_intensive_domain: + - data://garden/artificial_intelligence/2024-12-05/epoch_compute_intensive_domain ############### OTHERS ##################################################################################### diff --git a/dag/covid.yml b/dag/covid.yml index feaef3dae47..d0c1e5a290b 100644 --- a/dag/covid.yml +++ b/dag/covid.yml @@ -27,7 +27,7 @@ steps: - data://garden/regions/2023-01-01/regions # Demography - data://garden/demography/2024-07-15/population - - data://garden/demography/2023-10-09/life_expectancy + - data://garden/demography/2024-12-03/life_expectancy - data://garden/un/2024-07-12/un_wpp # Econ - data://garden/wb/2024-10-07/world_bank_pip @@ -325,4 +325,3 @@ steps: - grapher://grapher/covid/latest/covax - grapher://grapher/covid/latest/infections_model - grapher://grapher/covid/latest/xm_who - diff --git a/dag/demography.yml b/dag/demography.yml index 14880b11731..9fc85049335 100644 --- a/dag/demography.yml +++ b/dag/demography.yml @@ -1,7 +1,7 @@ steps: - #################################### - # Population ####################### - #################################### + ######################################################################## + # Population # + ######################################################################## # Population OMM ############################# # 2022-12-08: HYDE 3.2 + Gapminder + UN WPP 2022 @@ -88,13 +88,6 @@ steps: data://grapher/hyde/2024-01-02/all_indicators: - data://garden/hyde/2024-01-02/all_indicators - ## Population by education group (Wittgenstein centre) - data://meadow/demography/2023-04-07/population_education_wittgenstein: - - snapshot://demography/2023-04-07/population_education_wittgenstein.csv - data://garden/demography/2023-04-07/population_education_wittgenstein: - - data://meadow/demography/2023-04-07/population_education_wittgenstein - data://grapher/demography/2023-04-07/population_education_wittgenstein: - - data://garden/demography/2023-04-07/population_education_wittgenstein ## Population doubling times data://garden/demography/2024-07-18/population_doubling_times: - data://garden/demography/2024-07-15/population @@ -111,50 +104,21 @@ steps: data://grapher/demography/2023-07-03/world_population_comparison: - data://garden/demography/2023-06-27/world_population_comparison - #################################### - # Life expectancy ################## - #################################### - - # HMD - data://meadow/hmd/2023-09-19/hmd: - - snapshot://hmd/2023-09-18/hmd.zip - data://garden/hmd/2023-09-19/hmd: - - data://meadow/hmd/2023-09-19/hmd - data://grapher/hmd/2023-09-19/hmd: - - data://garden/hmd/2023-09-19/hmd - - # Gini Life Expectancy Inequality - data://garden/demography/2023-10-04/gini_le: - - data://garden/demography/2023-10-03/life_tables - data://grapher/demography/2023-10-04/gini_le: - - data://garden/demography/2023-10-04/gini_le - - # Survivorship ages (HMD-derived) - data://garden/demography/2023-09-27/survivor_percentiles: - - data://garden/hmd/2023-09-19/hmd - data://grapher/demography/2023-09-27/survivor_percentiles: - - data://garden/demography/2023-09-27/survivor_percentiles - - # Phi-gender life expectancy inequality - data://garden/demography/2023-10-03/phi_gender_le: - - data://garden/demography/2023-10-03/life_tables - data://grapher/demography/2023-10-03/phi_gender_le: - - data://garden/demography/2023-10-03/phi_gender_le + # Maddison working paper (2022) + data://meadow/ggdc/2024-01-19/maddison_federico_paper: + - snapshot://ggdc/2024-01-19/maddison_federico_paper.xlsx + data://garden/ggdc/2024-01-19/maddison_federico_paper: + - data://meadow/ggdc/2024-01-19/maddison_federico_paper - # UN WPP Life Tables - data://meadow/un/2023-10-02/un_wpp_lt: - - snapshot://un/2023-10-02/un_wpp_lt_all.zip - - snapshot://un/2023-10-02/un_wpp_lt_f.zip - - snapshot://un/2023-10-02/un_wpp_lt_m.zip - data://garden/un/2023-10-02/un_wpp_lt: - - data://meadow/un/2023-10-02/un_wpp_lt + # UN WPP largest age-group per country + data://garden/un/2024-03-14/un_wpp_most: + - data://garden/un/2024-07-12/un_wpp + data://grapher/un/2024-03-14/un_wpp_most: + - data://garden/un/2024-03-14/un_wpp_most - # UN WPP + HMD Life Tables - data://garden/demography/2023-10-03/life_tables: - - data://garden/hmd/2023-09-19/hmd - - data://garden/un/2023-10-02/un_wpp_lt - data://grapher/demography/2023-10-04/life_tables: - - data://garden/demography/2023-10-03/life_tables + ######################################################################## + # Life expectancy # + ######################################################################## # Zijdeman et al data://meadow/demography/2023-10-10/zijdeman_et_al_2015: @@ -168,14 +132,62 @@ steps: data://garden/demography/2023-10-10/riley_2005: - data://meadow/demography/2023-10-10/riley_2005 + # Human Mortality Database + data://meadow/hmd/2024-12-01/hmd: + - snapshot://hmd/2024-11-27/hmd.zip + data://garden/hmd/2024-12-01/hmd: + - data://meadow/hmd/2024-12-01/hmd + data://grapher/hmd/2024-12-01/hmd: + - data://garden/hmd/2024-12-01/hmd + + # UN WPP Life Tables + data://meadow/un/2024-12-02/un_wpp_lt: + - snapshot://un/2024-12-02/un_wpp_lt_m.csv + - snapshot://un/2024-12-02/un_wpp_lt_all.csv + - snapshot://un/2024-12-02/un_wpp_lt_f.csv + data://garden/un/2024-12-02/un_wpp_lt: + - data://meadow/un/2024-12-02/un_wpp_lt + + # Survivorship ages (HMD-derived) + data://garden/demography/2024-12-02/survivor_percentiles: + - data://garden/hmd/2024-12-01/hmd + data://grapher/demography/2024-12-02/survivor_percentiles: + - data://garden/demography/2024-12-02/survivor_percentiles + + # UN WPP + HMD Life Tables + data://garden/demography/2024-12-03/life_tables: + - data://garden/hmd/2024-12-01/hmd + - data://garden/un/2024-12-02/un_wpp_lt + data://grapher/demography/2024-12-03/life_tables: + - data://garden/demography/2024-12-03/life_tables + # OMM: Life Expectancy - data://garden/demography/2023-10-09/life_expectancy: - - data://garden/demography/2023-10-03/life_tables - - data://garden/demography/2023-10-10/zijdeman_et_al_2015 + data://garden/demography/2024-12-03/life_expectancy: - data://garden/demography/2023-10-10/riley_2005 - - data://garden/un/2022-07-11/un_wpp - data://grapher/demography/2023-10-10/life_expectancy: - - data://garden/demography/2023-10-09/life_expectancy + - data://garden/demography/2023-10-10/zijdeman_et_al_2015 + - data://garden/demography/2024-12-03/life_tables + - data://garden/un/2024-07-12/un_wpp + data://grapher/demography/2024-12-03/life_expectancy: + - data://garden/demography/2024-12-03/life_expectancy + + # Broken limits of Life Expectancy + data://garden/demography/2024-12-03/broken_limits_le: + - data://garden/hmd/2024-12-01/hmd + - data://garden/demography/2024-12-03/life_tables + data://grapher/demography/2024-12-03/broken_limits_le: + - data://garden/demography/2024-12-03/broken_limits_le + + # Phi-gender life expectancy inequality + data://garden/demography/2024-12-03/phi_gender_le: + - data://garden/demography/2024-12-03/life_tables + data://grapher/demography/2024-12-03/phi_gender_le: + - data://garden/demography/2024-12-03/phi_gender_le + + # Gini Life Expectancy Inequality + data://garden/demography/2024-12-03/gini_le: + - data://garden/demography/2024-12-03/life_tables + data://grapher/demography/2024-12-03/gini_le: + - data://garden/demography/2024-12-03/gini_le # Life Expectancy OECD data://meadow/oecd/2023-10-11/life_expectancy_birth: @@ -185,13 +197,6 @@ steps: data://grapher/oecd/2023-10-11/life_expectancy_birth: - data://garden/oecd/2023-10-11/life_expectancy_birth - # Broken limits of Life Expectancy - data://garden/demography/2023-10-20/broken_limits_le: - - data://garden/demography/2023-10-03/life_tables - - data://garden/hmd/2023-09-19/hmd - data://grapher/demography/2023-10-20/broken_limits_le: - - data://garden/demography/2023-10-20/broken_limits_le - # Contribution to sex gap in Life Expectancy data://meadow/demography/2023-11-08/le_sex_gap_age_contribution: - snapshot://demography/2023-11-08/le_sex_gap_age_contribution.zip @@ -208,61 +213,9 @@ steps: data://grapher/demography/2023-11-08/modal_age_death: - data://garden/demography/2023-11-08/modal_age_death - # Maddison working paper (2022) - data://meadow/ggdc/2024-01-19/maddison_federico_paper: - - snapshot://ggdc/2024-01-19/maddison_federico_paper.xlsx - data://garden/ggdc/2024-01-19/maddison_federico_paper: - - data://meadow/ggdc/2024-01-19/maddison_federico_paper - - # UN WPP experiments - data://garden/un/2024-03-14/un_wpp_most: - - data://garden/un/2022-07-11/un_wpp - data://grapher/un/2024-03-14/un_wpp_most: - - data://garden/un/2024-03-14/un_wpp_most - - # Migration UN DESA - data://meadow/un/2024-07-16/migrant_stock: - - snapshot://un/2024-07-16/migrant_stock_dest_origin.xlsx - - snapshot://un/2024-07-16/migrant_stock_origin.xlsx - - snapshot://un/2024-07-16/migrant_stock_dest.xlsx - - snapshot://un/2024-07-16/migrant_stock_age_sex.xlsx - data://garden/un/2024-07-16/migrant_stock: - - data://meadow/un/2024-07-16/migrant_stock - data://grapher/un/2024-07-16/migrant_stock: - - data://garden/un/2024-07-16/migrant_stock - - # Internal displacement monitoring centre - data://meadow/idmc/2024-08-02/internal_displacement: - - snapshot://idmc/2024-08-02/internal_displacement.xlsx - - data://garden/demography/2024-07-15/population - data://grapher/idmc/2024-08-02/internal_displacement: - - data://meadow/idmc/2024-08-02/internal_displacement - - # UNHCR refugee data - data://meadow/un/2024-07-25/refugee_data: - - snapshot://un/2024-07-25/refugee_data.zip - data://garden/un/2024-07-25/refugee_data: - - data://meadow/un/2024-07-25/refugee_data - - data://garden/demography/2024-07-15/population - - data://garden/un/2024-07-25/resettlement - data://grapher/un/2024-07-25/refugee_data: - - data://garden/un/2024-07-25/refugee_data - - # UNHCR resettlement data - data://meadow/un/2024-07-25/resettlement: - - snapshot://un/2024-07-25/resettlement.zip - data://garden/un/2024-07-25/resettlement: - - data://meadow/un/2024-07-25/resettlement - - data://garden/demography/2024-07-15/population - - # Child migration (UNICEF) - data://meadow/unicef/2024-07-30/child_migration: - - snapshot://unicef/2024-07-30/child_migration.csv - data://garden/unicef/2024-07-30/child_migration: - - data://meadow/unicef/2024-07-30/child_migration - - data://garden/demography/2024-07-15/population - data://grapher/unicef/2024-07-30/child_migration: - - data://garden/unicef/2024-07-30/child_migration + ######################################################################## + # Fertility # + ######################################################################## # Mothers by decadal age-group data://garden/un/2024-10-01/births_by_age: @@ -277,3 +230,69 @@ steps: - data://meadow/hmd/2024-11-19/hfd data://grapher/hmd/2024-11-19/hfd: - data://garden/hmd/2024-11-19/hfd + + # Multiple births + data://meadow/demography/2024-11-26/multiple_births: + - snapshot://demography/2024-11-26/multiple_births.7z + data://garden/demography/2024-11-26/multiple_births: + - data://meadow/demography/2024-11-26/multiple_births + data://grapher/demography/2024-11-26/multiple_births: + - data://garden/demography/2024-11-26/multiple_births + + # OMM: Fertility Rate -- HFD + UN WPP + data://garden/demography/2024-12-03/fertility_rate: + - data://garden/hmd/2024-11-19/hfd + - data://garden/un/2024-07-12/un_wpp + data://grapher/demography/2024-12-03/fertility_rate: + - data://garden/demography/2024-12-03/fertility_rate + + # OMM: Birth rate -- HFD + UN WPP + data://garden/demography/2024-12-03/birth_rate: + - data://garden/hmd/2024-12-01/hmd + - data://garden/un/2024-07-12/un_wpp + data://grapher/demography/2024-12-03/birth_rate: + - data://garden/demography/2024-12-03/birth_rate + + # HMD country data + data://meadow/hmd/2024-12-03/hmd_country: + - snapshot://hmd/2024-12-01/hmd_country.zip + + # HMD - Birth rate by month + data://garden/hmd/2024-12-03/hmd_country: + - data://meadow/hmd/2024-12-03/hmd_country + - data://garden/hmd/2024-12-01/hmd + data://grapher/hmd/2024-12-03/hmd_country: + - data://garden/hmd/2024-12-03/hmd_country + + # Patternal ages (Kai & Klüssner) + data://meadow/demography/2024-12-05/paternal_ages: + - snapshot://demography/2024-12-04/paternal_ages.rdata + data://garden/demography/2024-12-05/paternal_ages: + - data://meadow/demography/2024-12-05/paternal_ages + data://grapher/demography/2024-12-05/paternal_ages: + - data://garden/demography/2024-12-05/paternal_ages + + ######################################################################## + # OTHERS + ######################################################################## + + # Wittgenstein Centre (Projections) + data://meadow/demography/2024-12-06/wittgenstein_human_capital_proj: + - snapshot://demography/2024-12-06/wittgenstein_human_capital.zip + data://garden/demography/2024-12-06/wittgenstein_human_capital_proj: + - data://meadow/demography/2024-12-06/wittgenstein_human_capital_proj + # data://grapher/demography/2024-12-06/wittgenstein_human_capital_proj: + # - data://garden/demography/2024-12-06/wittgenstein_human_capital_proj + + # Wittgenstein Centre (Historical) + data://meadow/demography/2024-12-06/wittgenstein_human_capital_historical: + - snapshot://demography/2024-12-06/wittgenstein_human_capital_historical.zip + data://garden/demography/2024-12-06/wittgenstein_human_capital_historical: + - data://meadow/demography/2024-12-06/wittgenstein_human_capital_historical + + # Wittgenstein Centre (Projections + Historical) + data://garden/demography/2024-12-06/wittgenstein_human_capital: + - data://garden/demography/2024-12-06/wittgenstein_human_capital_historical + - data://garden/demography/2024-12-06/wittgenstein_human_capital_proj + data://grapher/demography/2024-12-06/wittgenstein_human_capital: + - data://garden/demography/2024-12-06/wittgenstein_human_capital diff --git a/dag/education.yml b/dag/education.yml index a52b02d95cc..b2384616d70 100644 --- a/dag/education.yml +++ b/dag/education.yml @@ -11,7 +11,6 @@ steps: data://grapher/education/2023-07-17/education_barro_lee_projections: - data://garden/education/2023-07-17/education_barro_lee_projections - # Barro and Lee historical estimates data://meadow/education/2023-07-17/education_lee_lee: - snapshot://education/2023-07-17/education_lee_lee.xlsx @@ -25,8 +24,7 @@ steps: data://grapher/education/2023-07-17/education_lee_lee: - data://garden/education/2023-07-17/education_lee_lee - -# OECD historical education data + # CLIO/INFRA data://meadow/education/2023-08-09/clio_infra_education: - snapshot://education/2023-08-09/years_of_education.xlsx - snapshot://education/2023-08-09/years_of_education_gini.xlsx @@ -38,6 +36,7 @@ steps: data://grapher/education/2023-08-09/clio_infra_education: - data://garden/education/2023-08-09/clio_infra_education + # OECD historical education data data://meadow/education/2023-08-14/oecd_education: - snapshot://education/2023-08-14/oecd_education.csv data://garden/education/2023-08-14/oecd_education: @@ -46,6 +45,7 @@ steps: data://grapher/education/2023-08-14/oecd_education: - data://garden/education/2023-08-14/oecd_education + # ILOstat Unemployment data://meadow/ilostat/2023-09-19/unemployment: - snapshot://ilostat/2023-09-19/unemployment.csv data://garden/ilostat/2023-09-19/unemployment: @@ -53,6 +53,7 @@ steps: data://grapher/ilostat/2023-09-19/unemployment: - data://garden/ilostat/2023-09-19/unemployment + # ILOstat Employment data://meadow/ilostat/2023-09-19/employment: - snapshot://ilostat/2023-09-19/employment.csv data://garden/ilostat/2023-09-19/employment: @@ -60,6 +61,7 @@ steps: data://grapher/ilostat/2023-09-19/employment: - data://garden/ilostat/2023-09-19/employment + # ILOstat Average Work Hours data://meadow/ilostat/2023-09-19/average_work_hours: - snapshot://ilostat/2023-09-19/average_work_hours.csv data://garden/ilostat/2023-09-19/average_work_hours: @@ -80,15 +82,12 @@ steps: data://grapher/oecd/2023-12-06/pisa: - data://garden/oecd/2023-12-06/pisa - # Wittgenstein Center Population and Human Capital Projections - Educational Attainment - data://meadow/education/2023-12-15/wittgenstein_center: - - snapshot://education/2023-12-15/wittgenstein_center_data.csv - - snapshot://education/2023-12-15/wittgenstein_center_dictionary.csv - data://garden/education/2023-12-15/wittgenstein_center: - - data://meadow/education/2023-12-15/wittgenstein_center + # OECD + Wittgenstein: People with education (long-run) + data://garden/education/2024-12-11/people_with_education: + - data://garden/demography/2024-12-06/wittgenstein_human_capital - data://garden/education/2023-08-14/oecd_education - data://grapher/education/2023-12-15/wittgenstein_center: - - data://garden/education/2023-12-15/wittgenstein_center + data://grapher/education/2024-12-11/people_with_education: + - data://garden/education/2024-12-11/people_with_education # UNESCO data on other policy related education indicators data://meadow/unesco/2024-06-16/education_opri: @@ -117,7 +116,6 @@ steps: data://grapher/unesco/2024-11-21/enrolment_rates: - data://garden/unesco/2024-11-21/enrolment_rates - # World Bank EdStats data://meadow/wb/2024-11-04/edstats: - snapshot://wb/2024-11-04/edstats.csv diff --git a/dag/energy.yml b/dag/energy.yml index 0ec3bdb2621..f9d129f15d5 100644 --- a/dag/energy.yml +++ b/dag/energy.yml @@ -228,3 +228,54 @@ steps: # data://grapher/energy/2024-11-15/photovoltaic_cost_and_capacity: - data://garden/energy/2024-11-15/photovoltaic_cost_and_capacity + # + # Eurostat - Energy statistics, prices of natural gas and electricity + # + data://meadow/eurostat/2024-11-05/gas_and_electricity_prices: + - snapshot://eurostat/2024-11-05/gas_and_electricity_prices.zip + # + # Eurostat - Energy statistics, prices of natural gas and electricity + # + data://garden/eurostat/2024-11-05/gas_and_electricity_prices: + - data://meadow/eurostat/2024-11-05/gas_and_electricity_prices + # + # Ember - European wholesale electricity prices + # + data://meadow/ember/2024-11-20/european_wholesale_electricity_prices: + - snapshot://ember/2024-11-20/european_wholesale_electricity_prices.csv + # + # Ember - European wholesale electricity prices + # + data://garden/ember/2024-11-20/european_wholesale_electricity_prices: + - data://meadow/ember/2024-11-20/european_wholesale_electricity_prices + # + # IEA - Fossil fuel subsidies + # + data://meadow/iea/2024-11-20/fossil_fuel_subsidies: + - snapshot://iea/2024-11-20/fossil_fuel_subsidies.xlsx + # + # IEA - Fossil fuel subsidies + # + data://garden/iea/2024-11-20/fossil_fuel_subsidies: + - data://meadow/iea/2024-11-20/fossil_fuel_subsidies + # + # IEA - Fossil fuel subsidies + # + data://grapher/iea/2024-11-20/fossil_fuel_subsidies: + - data://garden/iea/2024-11-20/fossil_fuel_subsidies + # + # Energy prices + # + data://garden/energy/2024-11-20/energy_prices: + - data://garden/eurostat/2024-11-05/gas_and_electricity_prices + - data://garden/ember/2024-11-20/european_wholesale_electricity_prices + # + # Energy prices + # + data://grapher/energy/2024-11-20/energy_prices: + - data://garden/energy/2024-11-20/energy_prices + # + # Energy prices explorer + # + export://multidim/energy/latest/energy_prices: + - data://grapher/energy/2024-11-20/energy_prices diff --git a/dag/fasttrack.yml b/dag/fasttrack.yml index 607caf19b2f..2c0f8c90876 100644 --- a/dag/fasttrack.yml +++ b/dag/fasttrack.yml @@ -74,8 +74,6 @@ steps: - snapshot-private://fasttrack/latest/pain_hours_hen_systems.csv data-private://grapher/fasttrack/latest/antibiotic_usage_livestock: - snapshot-private://fasttrack/latest/antibiotic_usage_livestock.csv - data-private://grapher/fasttrack/latest/antimicrobial_usage_livestock: - - snapshot-private://fasttrack/latest/antimicrobial_usage_livestock.csv data://grapher/fasttrack/2023-08-07/pain_hours_days_hen_systems: - snapshot://fasttrack/2023-08-07/pain_hours_days_hen_systems.csv data-private://grapher/fasttrack/latest/historical_france_mortality_cause: @@ -236,7 +234,9 @@ steps: - snapshot://fasttrack/latest/pepfar_patients_receiving_art.csv data://grapher/fasttrack/latest/simon_ehrlich_wager: - snapshot://fasttrack/latest/simon_ehrlich_wager.csv - data://grapher/fasttrack/latest/mineral_prices_usgs: - - snapshot://fasttrack/latest/mineral_prices_usgs.csv data://grapher/fasttrack/latest/useful_energy_cost_way: - snapshot://fasttrack/latest/useful_energy_cost_way.csv + data://grapher/fasttrack/2023-06-19/world_population_comparison: + - snapshot://fasttrack/2023-06-19/world_population_comparison.csv + data://grapher/fasttrack/latest/antimicrobial_usage_livestock: + - snapshot://fasttrack/latest/antimicrobial_usage_livestock.csv diff --git a/dag/health.yml b/dag/health.yml index 268ca8d9c25..2e47cb03825 100644 --- a/dag/health.yml +++ b/dag/health.yml @@ -39,7 +39,6 @@ steps: data://grapher/who/2023-03-09/gho_suicides: - data://garden/who/2023-03-09/gho_suicides - # IHME Global Burden of Disease - Risk factors data://meadow/ihme_gbd/2019/gbd_risk: - walden://ihme_gbd/2019/gbd_risk @@ -48,8 +47,6 @@ steps: data://grapher/ihme_gbd/2019/gbd_risk: - data://garden/ihme_gbd/2019/gbd_risk - - # IHME GBD Leading cause of deaths - update data-private://meadow/ihme_gbd/2024-06-10/cause_hierarchy: - snapshot-private://ihme_gbd/2024-06-10/cause_hierarchy.csv @@ -192,7 +189,6 @@ steps: data://grapher/un/2023-08-10/comtrade_pandemics: - data://garden/un/2023-08-02/comtrade_pandemics - # UNAIDS data://meadow/health/2023-08-09/unaids: - snapshot://health/2023-08-09/unaids.csv @@ -232,7 +228,6 @@ steps: data://grapher/oecd/2024-07-01/road_accidents: - data://garden/oecd/2024-07-01/road_accidents - # Kucharski data://meadow/health/2023-08-14/avian_influenza_h5n1_kucharski: - snapshot://health/2023-08-14/avian_influenza_h5n1_kucharski.xlsx @@ -577,7 +572,6 @@ steps: data-private://grapher/ihme_gbd/2024-05-20/gbd_mental_health_burden_dalys: - data-private://garden/ihme_gbd/2024-05-20/gbd_mental_health_burden - # GBD 2021 - GBD Risk Factors data-private://meadow/ihme_gbd/2024-05-20/gbd_risk: - snapshot-private://ihme_gbd/2024-05-20/gbd_risk.feather @@ -598,7 +592,6 @@ steps: data-private://grapher/ihme_gbd/2024-05-20/gbd_drug_risk: - data-private://garden/ihme_gbd/2024-05-20/gbd_drug_risk - # GBD 2021 - GBD Child Mortality data-private://meadow/ihme_gbd/2024-05-20/gbd_child_mortality: - snapshot-private://ihme_gbd/2024-05-20/gbd_child_mortality.feather @@ -609,7 +602,6 @@ steps: data-private://grapher/ihme_gbd/2024-05-20/gbd_child_mortality: - data-private://garden/ihme_gbd/2024-05-20/gbd_child_mortality - # GBD 2021 - GBD Health-adjusted Life Expectancy and Life Expectancy data-private://meadow/ihme_gbd/2024-07-02/gbd_life_expectancy: - snapshot-private://ihme_gbd/2024-07-02/gbd_life_expectancy.zip @@ -715,7 +707,6 @@ steps: data://grapher/who/2024-08-06/mortality_database_cancer_most_common: - data://garden/who/2024-08-06/mortality_database_cancer_most_common - data://meadow/who/latest/monkeypox: - snapshot://who/latest/monkeypox.csv data://garden/who/latest/monkeypox: @@ -729,7 +720,7 @@ steps: - data://garden/who/latest/monkeypox export://github/who/latest/monkeypox: - data://garden/who/latest/monkeypox -# Mpox - Global.health + # Mpox - Global.health data://meadow/health/latest/global_health_mpox: - snapshot://health/latest/global_health_mpox.csv data://garden/health/latest/global_health_mpox: @@ -737,7 +728,7 @@ steps: # Eurostat cancer - # Eurostat Cancer Screening + # Eurostat Cancer Screening data://meadow/health/2024-08-23/eurostat_cancer: - snapshot://health/2024-08-23/eurostat_cancer.csv data://garden/health/2024-08-23/eurostat_cancer: @@ -745,12 +736,10 @@ steps: data://grapher/health/2024-08-23/eurostat_cancer: - data://garden/health/2024-08-23/eurostat_cancer - # Multi-dim indicators export://multidim/health/latest/causes_of_death: - grapher://grapher/ihme_gbd/2024-05-20/gbd_cause - # GBD 2021 - GBD Risk Factors cancer specific data-private://meadow/ihme_gbd/2024-08-26/gbd_risk_cancer: - snapshot-private://ihme_gbd/2024-08-26/gbd_risk_cancer.feather @@ -775,7 +764,6 @@ steps: data://grapher/health/2024-09-05/seattle_pathogens: - data://garden/health/2024-09-05/seattle_pathogens - # International Agency for Research on Cancer data://meadow/cancer/2024-08-30/gco_alcohol: - snapshot://cancer/2024-08-30/gco_alcohol.csv @@ -791,13 +779,13 @@ steps: data://grapher/cancer/2024-09-06/gco_infections: - data://garden/cancer/2024-09-06/gco_infections -# Flu testing data + # Flu testing data data://garden/who/2024-09-09/flu_test: - data://meadow/who/latest/flunet data://grapher/who/2024-09-09/flu_test: - data://garden/who/2024-09-09/flu_test -# Cancer diagnosis routes and survival rates + # Cancer diagnosis routes and survival rates data://meadow/cancer/2024-09-13/diagnosis_routes_by_route: - snapshot://cancer/2024-09-13/diagnosis_routes_by_route.csv data://garden/cancer/2024-09-13/diagnosis_routes_by_route: @@ -847,7 +835,6 @@ steps: data://grapher/antibiotics/2024-10-09/gram_children: - data://garden/antibiotics/2024-10-09/gram_children - # Cervical cancer incidence rates GCO - Cancer Over Time data://meadow/cancer/2024-10-13/gco_cancer_over_time_cervical: - snapshot://cancer/2024-10-13/gco_cancer_over_time_cervical.csv @@ -888,7 +875,6 @@ steps: data://grapher/antibiotics/2024-10-23/animuse_year: - data://garden/antibiotics/2024-10-23/animuse_year - # ESVAC antimicrobial use in animals data://meadow/antibiotics/2024-10-25/esvac_sales: - snapshot://antibiotics/2024-10-25/esvac_sales.zip @@ -939,19 +925,81 @@ steps: - snapshot-private://antibiotics/2024-11-20/pathogen_bloodstream.csv data-private://garden/antibiotics/2024-11-20/pathogen_bloodstream: - data-private://meadow/antibiotics/2024-11-20/pathogen_bloodstream - data-private://grapher/antibiotics/2024-11-20/pathogen_bloodstream: - - data-private://garden/antibiotics/2024-11-20/pathogen_bloodstream - # IHME Neonatal bloodstream infections by resistance - data-private://meadow/antibiotics/2024-11-20/bloodstream_amr: - - snapshot-private://antibiotics/2024-11-20/bloodstream_amr.csv - data-private://garden/antibiotics/2024-11-20/bloodstream_amr: - - data-private://meadow/antibiotics/2024-11-20/bloodstream_amr - data-private://grapher/antibiotics/2024-11-20/bloodstream_amr: - - data-private://garden/antibiotics/2024-11-20/bloodstream_amr + # IHME Neonatal infections by syndrome data-private://meadow/antibiotics/2024-11-24/total_syndrome: - snapshot-private://antibiotics/2024-11-24/total_syndrome.csv - data-private://garden/antibiotics/2024-11-24/total_syndrome: + # IHME Neonatal infections by syndrome and amr resistance + data-private://meadow/antibiotics/2024-12-02/microbe_amr: + - snapshot-private://antibiotics/2024-12-02/microbe_amr.csv + data-private://garden/antibiotics/2024-12-02/microbe_amr: + - data-private://meadow/antibiotics/2024-12-02/microbe_amr - data-private://meadow/antibiotics/2024-11-24/total_syndrome - data-private://grapher/antibiotics/2024-11-24/total_syndrome: - - data-private://garden/antibiotics/2024-11-24/total_syndrome + data-private://grapher/antibiotics/2024-12-02/microbe_amr: + - data-private://garden/antibiotics/2024-12-02/microbe_amr + # IHME Neonatal infections and amr resistance + data-private://meadow/antibiotics/2024-12-02/microbe_neonatal_amr: + - snapshot-private://antibiotics/2024-12-02/microbe_neonatal_amr.csv + data-private://garden/antibiotics/2024-12-02/microbe_neonatal_amr: + - data-private://meadow/antibiotics/2024-12-02/microbe_neonatal_amr + - data-private://meadow/antibiotics/2024-11-20/microbe + data-private://grapher/antibiotics/2024-12-02/microbe_neonatal_amr: + - data-private://garden/antibiotics/2024-12-02/microbe_neonatal_amr + + # MICROBE - total deaths by pathogen + data-private://meadow/antibiotics/2024-12-02/total_pathogen_bloodstream: + - snapshot-private://antibiotics/2024-12-02/total_pathogen_bloodstream.csv + data-private://garden/antibiotics/2024-12-02/total_pathogen_bloodstream: + - data-private://meadow/antibiotics/2024-12-02/total_pathogen_bloodstream + + # MICROBE - total deaths by pathogen and amr resistance + data-private://meadow/antibiotics/2024-12-02/total_pathogen_bloodstream_amr: + - snapshot-private://antibiotics/2024-12-02/total_pathogen_bloodstream_amr.csv + data-private://garden/antibiotics/2024-12-02/total_pathogen_bloodstream_amr: + - data-private://garden/antibiotics/2024-12-02/total_pathogen_bloodstream + - data-private://meadow/antibiotics/2024-12-02/total_pathogen_bloodstream_amr + data-private://grapher/antibiotics/2024-12-02/total_pathogen_bloodstream_amr: + - data-private://garden/antibiotics/2024-12-02/total_pathogen_bloodstream_amr + # WHO GLASS Enrolment + data://meadow/antibiotics/2024-12-03/glass_enrolment: + - snapshot://antibiotics/2024-12-03/glass_enrolment.xlsx + data://garden/antibiotics/2024-12-03/glass_enrolment: + - data://meadow/antibiotics/2024-12-03/glass_enrolment + data://grapher/antibiotics/2024-12-03/glass_enrolment: + - data://garden/antibiotics/2024-12-03/glass_enrolment + # MICROBE - total deaths by pathogen + data-private://meadow/antibiotics/2024-12-04/microbe_total_pathogens: + - snapshot-private://antibiotics/2024-12-04/microbe_total_pathogens.csv + data-private://garden/antibiotics/2024-12-04/microbe_total_pathogens: + - data-private://meadow/antibiotics/2024-12-04/microbe_total_pathogens + # MICROBE - total deaths by pathogen and amr resistance + data-private://meadow/antibiotics/2024-12-04/microbe_total_pathogens_amr: + - snapshot-private://antibiotics/2024-12-04/microbe_total_pathogens_amr.csv + data-private://garden/antibiotics/2024-12-04/microbe_total_pathogens_amr: + - data-private://meadow/antibiotics/2024-12-04/microbe_total_pathogens_amr + - data-private://garden/antibiotics/2024-12-04/microbe_total_pathogens + data-private://grapher/antibiotics/2024-12-04/microbe_total_pathogens_amr: + - data-private://garden/antibiotics/2024-12-04/microbe_total_pathogens_amr + + # MICROBE - neonatal deaths by pathogen and amr resistance + data-private://meadow/antibiotics/2024-12-05/microbe_neonatal_total_amr: + - snapshot-private://antibiotics/2024-12-05/microbe_neonatal_total_amr.csv + data-private://garden/antibiotics/2024-12-05/microbe_neonatal_total_amr: + - data-private://meadow/antibiotics/2024-12-05/microbe_neonatal_total_amr + - data-private://garden/antibiotics/2024-11-20/microbe + data-private://grapher/antibiotics/2024-12-05/microbe_neonatal_total_amr: + - data-private://garden/antibiotics/2024-12-05/microbe_neonatal_total_amr + # MICROBE - total deaths by syndrome + data-private://meadow/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome: + - snapshot-private://antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.csv + data-private://garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome: + - data-private://meadow/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome + + # MICROBE - total deaths by syndrome and amr resistance + data-private://meadow/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr: + - snapshot-private://antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.csv + data-private://garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr: + - data-private://meadow/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr + - data-private://garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome + data-private://grapher/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr: + - data-private://garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr diff --git a/dag/main.yml b/dag/main.yml index 7d4f3ee84bb..99a5ec05481 100644 --- a/dag/main.yml +++ b/dag/main.yml @@ -100,7 +100,6 @@ steps: data://grapher/homicide/2024-10-30/unodc: - data://garden/homicide/2024-10-30/unodc - # # UN data://grapher/un/2022-07-11/un_wpp: @@ -131,11 +130,6 @@ steps: - data://garden/regions/2023-01-01/regions data://grapher/technology/2022/internet: - data://garden/technology/2022/internet - # HMD - data://meadow/hmd/2022-12-07/life_tables: - - snapshot://hmd/2022-12-07/hmd.zip - data://garden/hmd/2022-12-07/life_tables: - - data://meadow/hmd/2022-12-07/life_tables # UNDP data://meadow/un/2024-04-09/undp_hdr: @@ -321,8 +315,6 @@ steps: data://grapher/terrorism/2023-07-20/global_terrorism_database: - data://garden/terrorism/2023-07-20/global_terrorism_database - - # Colonial Dates Dataset (COLDAT) data://meadow/harvard/2023-09-18/colonial_dates_dataset: - snapshot://harvard/2023-09-18/colonial_dates_dataset.csv @@ -355,7 +347,6 @@ steps: data://garden/gapminder/2023-09-21/under_five_mortality: - data://meadow/gapminder/2023-09-21/under_five_mortality - # Older vintage UN IGME (with longer time-series) data://meadow/un/2018/igme: - snapshot://un/2018/igme.csv @@ -763,7 +754,7 @@ steps: data-private://grapher/language/2024-07-17/ethnologue: - data-private://garden/language/2024-07-17/ethnologue -# Child Mortality Estimates - UN IGME + # Child Mortality Estimates - UN IGME data://meadow/un/2024-09-11/igme: - snapshot://un/2024-09-11/igme.zip data://garden/un/2024-09-11/igme: @@ -774,8 +765,7 @@ steps: data://grapher/un/2024-09-11/igme: - data://garden/un/2024-09-11/igme - -# Long-run child mortality, Gapminder + UN IGME + # Long-run child mortality, Gapminder + UN IGME data://garden/un/2024-09-16/long_run_child_mortality: - data://garden/un/2024-09-11/igme - data://garden/gapminder/2023-09-18/under_five_mortality @@ -783,8 +773,7 @@ steps: data://grapher/un/2024-09-16/long_run_child_mortality: - data://garden/un/2024-09-16/long_run_child_mortality - -# UN SDG (2024) + # UN SDG (2024) data://meadow/un/2024-08-27/un_sdg: - snapshot://un/2024-08-27/un_sdg.feather data://garden/un/2024-08-27/un_sdg: @@ -794,7 +783,7 @@ steps: data://grapher/un/2024-08-27/un_sdg: - data://garden/un/2024-08-27/un_sdg -# OECD Official Development Assistance (ODA) + # OECD Official Development Assistance (ODA) data://meadow/oecd/2024-08-21/official_development_assistance: - snapshot://oecd/2024-08-21/official_development_assistance_dac1.zip - snapshot://oecd/2024-08-21/official_development_assistance_dac2a.zip @@ -806,7 +795,7 @@ steps: data://grapher/oecd/2024-08-21/official_development_assistance: - data://garden/oecd/2024-08-21/official_development_assistance -# Oil Spills + # Oil Spills data://meadow/itopf/2024-10-16/oil_spills: - snapshot://itopf/2024-10-16/oil_spills.pdf data://garden/itopf/2024-10-16/oil_spills: @@ -814,7 +803,7 @@ steps: data://grapher/itopf/2024-10-16/oil_spills: - data://garden/itopf/2024-10-16/oil_spills -# UN SD census data + # UN SD census data data://meadow/un/2024-10-21/census_dates: - snapshot://un/2024-10-21/census_dates.csv data://garden/un/2024-10-21/census_dates: @@ -822,7 +811,7 @@ steps: data://grapher/un/2024-10-21/census_dates: - data://garden/un/2024-10-21/census_dates -# World Peace Foundation - Famines + # World Peace Foundation - Famines data://meadow/wpf/2024-10-03/famines: - snapshot://wpf/2024-10-03/famines.xlsx data://garden/wpf/2024-10-03/famines: @@ -861,7 +850,14 @@ steps: data-private://grapher/owid/latest/ig_countries: - data-private://garden/owid/latest/ig_countries -# Migration between regions, based on UN DESA flows +# Migration distances + data://garden/migration/2024-11-19/migration_distance: + - data://garden/un/2024-07-16/migrant_stock + - data://garden/geography/2023-11-28/nat_earth_110 + data://grapher/migration/2024-11-19/migration_distance: + - data://garden/migration/2024-11-19/migration_distance + + # Migration between regions, based on UN DESA flows data://garden/migration/2024-11-18/migration_between_regions: - data://garden/un/2024-07-16/migrant_stock - data://garden/regions/2023-01-01/regions @@ -878,6 +874,7 @@ steps: data://grapher/imf/2024-11-25/world_economic_outlook: - data://garden/imf/2024-11-25/world_economic_outlook + include: - dag/open_numbers.yml - dag/faostat.yml @@ -913,3 +910,4 @@ include: - dag/chartbook.yml - dag/minerals.yml - dag/tourism.yml + - dag/migration.yml diff --git a/dag/migration.yml b/dag/migration.yml new file mode 100644 index 00000000000..3167519dffa --- /dev/null +++ b/dag/migration.yml @@ -0,0 +1,44 @@ +steps: + # Migration UN DESA + data://meadow/un/2024-07-16/migrant_stock: + - snapshot://un/2024-07-16/migrant_stock_dest_origin.xlsx + - snapshot://un/2024-07-16/migrant_stock_origin.xlsx + - snapshot://un/2024-07-16/migrant_stock_dest.xlsx + - snapshot://un/2024-07-16/migrant_stock_age_sex.xlsx + data://garden/un/2024-07-16/migrant_stock: + - data://meadow/un/2024-07-16/migrant_stock + data://grapher/un/2024-07-16/migrant_stock: + - data://garden/un/2024-07-16/migrant_stock + + # Internal displacement monitoring centre + data://meadow/idmc/2024-08-02/internal_displacement: + - snapshot://idmc/2024-08-02/internal_displacement.xlsx + - data://garden/demography/2024-07-15/population + data://grapher/idmc/2024-08-02/internal_displacement: + - data://meadow/idmc/2024-08-02/internal_displacement + + # UNHCR refugee data + data://meadow/un/2024-07-25/refugee_data: + - snapshot://un/2024-07-25/refugee_data.zip + data://garden/un/2024-07-25/refugee_data: + - data://meadow/un/2024-07-25/refugee_data + - data://garden/demography/2024-07-15/population + - data://garden/un/2024-07-25/resettlement + data://grapher/un/2024-07-25/refugee_data: + - data://garden/un/2024-07-25/refugee_data + + # UNHCR resettlement data + data://meadow/un/2024-07-25/resettlement: + - snapshot://un/2024-07-25/resettlement.zip + data://garden/un/2024-07-25/resettlement: + - data://meadow/un/2024-07-25/resettlement + - data://garden/demography/2024-07-15/population + + # Child migration (UNICEF) + data://meadow/unicef/2024-07-30/child_migration: + - snapshot://unicef/2024-07-30/child_migration.csv + data://garden/unicef/2024-07-30/child_migration: + - data://meadow/unicef/2024-07-30/child_migration + - data://garden/demography/2024-07-15/population + data://grapher/unicef/2024-07-30/child_migration: + - data://garden/unicef/2024-07-30/child_migration diff --git a/dag/minerals.yml b/dag/minerals.yml index 82a8d9ec567..85df4de7416 100644 --- a/dag/minerals.yml +++ b/dag/minerals.yml @@ -68,6 +68,11 @@ steps: data://grapher/minerals/2024-07-15/minerals: - data://garden/minerals/2024-07-15/minerals # + # Minerals - Minerals. + # + data://grapher/minerals/2024-07-15/global_mine_production_by_mineral: + - data://garden/minerals/2024-07-15/minerals + # # Minerals - Minerals explorer. # export://explorers/minerals/latest/minerals: diff --git a/dag/poverty_inequality.yml b/dag/poverty_inequality.yml index 0e8c94291c3..25a1c80a547 100644 --- a/dag/poverty_inequality.yml +++ b/dag/poverty_inequality.yml @@ -114,15 +114,6 @@ steps: data://grapher/oecd/2024-04-30/affordable_housing_database: - data://garden/oecd/2024-04-30/affordable_housing_database - # Poverty projections from the World Bank - data://meadow/wb/2024-06-26/poverty_projections: - - snapshot://wb/2024-06-26/poverty_projections_number_global.csv - - snapshot://wb/2024-06-26/poverty_projections_share_regions.csv - data://garden/wb/2024-06-26/poverty_projections: - - data://meadow/wb/2024-06-26/poverty_projections - data://grapher/wb/2024-06-26/poverty_projections: - - data://garden/wb/2024-06-26/poverty_projections - # Institute of Global Homelessness - Better Data Project data://meadow/igh/2024-07-05/better_data_homelessness: - snapshot://igh/2024-07-05/better_data_homelessness.xlsx @@ -130,3 +121,11 @@ steps: - data://meadow/igh/2024-07-05/better_data_homelessness data://grapher/igh/2024-07-05/better_data_homelessness: - data://garden/igh/2024-07-05/better_data_homelessness + + # Poverty projections from the Poverty, Prosperity and Planet Report 2024 + data://meadow/wb/2024-12-03/poverty_projections: + - snapshot://wb/2024-12-03/reproducibility_package_poverty_prosperity_planet.zip + data://garden/wb/2024-12-03/poverty_projections: + - data://meadow/wb/2024-12-03/poverty_projections + data://grapher/wb/2024-12-03/poverty_projections: + - data://garden/wb/2024-12-03/poverty_projections diff --git a/dag/urbanization.yml b/dag/urbanization.yml index c4c4fdc49f7..ecff7c1f0a8 100644 --- a/dag/urbanization.yml +++ b/dag/urbanization.yml @@ -45,17 +45,6 @@ steps: data://grapher/un/2024-01-17/urban_agglomerations_definition_count: - data://garden/un/2024-01-17/urban_agglomerations_definition_count # - # GHSL degree of urbanization. - # - data://meadow/urbanization/2024-01-26/ghsl_degree_of_urbanisation: - - snapshot://urbanization/2024-01-26/ghsl_degree_of_urbanisation.zip - data://garden/urbanization/2024-01-26/ghsl_degree_of_urbanisation: - - data://meadow/urbanization/2024-01-26/ghsl_degree_of_urbanisation - - data://garden/wb/2023-04-30/income_groups - - data://garden/regions/2023-01-01/regions - data://grapher/urbanization/2024-01-26/ghsl_degree_of_urbanisation: - - data://garden/urbanization/2024-01-26/ghsl_degree_of_urbanisation - # # UN SDG indicators related to urbanization. # data://meadow/un/2024-02-14/sdgs_urbanization: @@ -76,3 +65,13 @@ steps: - data://garden/regions/2023-01-01/regions data://grapher/urbanization/2024-10-14/ghsl_degree_of_urbanisation: - data://garden/urbanization/2024-10-14/ghsl_degree_of_urbanisation + + # GHSL urban centers. + data://meadow/urbanization/2024-12-02/ghsl_urban_centers: + - snapshot://urbanization/2024-12-02/ghsl_urban_centers.xlsx + data://garden/urbanization/2024-12-02/ghsl_urban_centers: + - data://meadow/urbanization/2024-12-02/ghsl_urban_centers + - data://garden/wb/2024-07-29/income_groups + - data://garden/regions/2023-01-01/regions + data://grapher/urbanization/2024-12-02/ghsl_urban_centers: + - data://garden/urbanization/2024-12-02/ghsl_urban_centers diff --git a/docs/architecture/metadata/structuring-yaml.md b/docs/architecture/metadata/structuring-yaml.md index bb8a16b2862..f0c4091c5da 100644 --- a/docs/architecture/metadata/structuring-yaml.md +++ b/docs/architecture/metadata/structuring-yaml.md @@ -233,4 +233,41 @@ tables: {definitions.conflict_type_estimate} ``` -Be cautious with line breaks and trailing whitespace when utilizing templates. Despite using good defaults, you might end up experimenting a lot to get the desired result. +Line breaks and whitespaces can be tricky when using Jinja templates. We use reasonable defaults and strip whitespaces, so in most cases you should be fine with using `<%` and `%>`, but in more complex cases, you might have to experiment with +more fine grained [whitespace control](https://jinja.palletsprojects.com/en/stable/templates/#whitespace-control) using tags `<%-` and `-%>`. This is most often used in if-else blocks like this + +```yaml +age: |- + <% if age_group == "ALLAges" %> + ... + <%- elif age_group == "Age-standardized" %> + ... + <%- else %> + ... + <%- endif %> +``` + +The most straightforward way to check your metadata is in Admin, although that means waiting for your step to finish. There's a faster way to check your YAML file directly. Create a `playground.ipynb` notebook in the same folder as your YAML file and copy this to the first cell: + +```python +from etl import grapher_helpers as gh +dim_dict = { + "age_group": "YEARS0-4", "sex": "Male", "cause": "Drug use disorders" +} +d = gh.render_yaml_file("ghe.meta.yml", dim_dict=dim_dict) +d["tables"]["ghe"]["variables"]["death_count"] +``` + +An alternative is examining `VariableMeta` + +```python +from etl import grapher_helpers as gh +from etl import paths + +tb = Dataset(paths.DATA_DIR / "garden/who/2024-07-30/ghe")['ghe'] + +# Sample a random row to get the dimension values +dim_dict = dict(zip(tb.index.names, tb.sample(1).index[0])) + +gh.render_variable_meta(tb.death_count.m, dim_dict=dim_dict) +``` diff --git a/docs/assets/annotations-chart.png b/docs/assets/annotations-chart.png new file mode 100644 index 00000000000..6d4383983f2 Binary files /dev/null and b/docs/assets/annotations-chart.png differ diff --git a/docs/guides/data-work/add-data.md b/docs/guides/data-work/add-data.md index b3fdbe7aff5..e42cf58169c 100644 --- a/docs/guides/data-work/add-data.md +++ b/docs/guides/data-work/add-data.md @@ -39,7 +39,7 @@ There are different ways you can add data to the catalog, depending on your tech Before starting to add a new dataset, make sure to create your new environment. This means creating a new branch, its corresponding pull request and staging server. This can all be made with one command: ```bash -etl pr data-{short_name} -c data -t "{short_name}: new dataset" +etl pr "{short_name}: new dataset" data ``` This will create a new git branch in your local repository with an empty commit, which will be pushed to remote. It will also create a draft pull request in github, and a staging server. Wait for a notification from [@owidbot](https://github.com/owidbot). It should take a few minutes, and will inform you that the staging server `http://staging-site-data-{short_name}` has been created. diff --git a/docs/guides/data-work/export-data.md b/docs/guides/data-work/export-data.md index a3ff3341601..8dbe49b9ae2 100644 --- a/docs/guides/data-work/export-data.md +++ b/docs/guides/data-work/export-data.md @@ -31,101 +31,93 @@ ds_explorer.save() Multi-dimensional indicators are powered by a configuration that is typically created from a YAML file. The structure of the YAML file looks like this: -```yaml title="etl/steps/export/multidim/covid/latest/covid.deaths.yaml" -definitions: - table: {definitions.table} - +```yaml title="etl/steps/export/multidim/energy/latest/energy_prices.yaml" title: - title: COVID-19 deaths - titleVariant: by interval + title: "Energy prices" + titleVariant: "by energy source" defaultSelection: - - World - - Europe - - Asia + - "European Union (27)" topicTags: - - COVID-19 - + - "Energy" dimensions: - - slug: interval - name: Interval + - slug: "frequency" + name: "Frequency" choices: - - slug: weekly - name: Weekly - description: null - - slug: biweekly - name: Biweekly - description: null - - - slug: metric - name: Metric + - slug: "annual" + name: "Annual" + description: "Annual data" + - slug: "monthly" + name: "Monthly" + description: "Monthly data" + - slug: "source" + name: "Energy source" choices: - - slug: absolute - name: Absolute - description: null - - slug: per_capita - name: Per million people - description: null - - slug: change - name: Change from previous interval - description: null - + - slug: "electricity" + name: "Electricity" + - slug: "gas" + name: "Gas" + - slug: "unit" + name: "Unit" + choices: + - slug: "euro" + name: "Euro" + description: "Price in euros" + - slug: "pps" + name: "PPS" + description: "Price in Purchasing Power Standard" views: - - dimensions: - interval: weekly - metric: absolute - indicators: - y: "{definitions.table}#weekly_deaths" - - dimensions: - interval: weekly - metric: per_capita - indicators: - y: "{definitions.table}#weekly_deaths_per_million" - - dimensions: - interval: weekly - metric: change - indicators: - y: "{definitions.table}#weekly_pct_growth_deaths" - - - dimensions: - interval: biweekly - metric: absolute - indicators: - y: "{definitions.table}#biweekly_deaths" - - dimensions: - interval: biweekly - metric: per_capita - indicators: - y: "{definitions.table}#biweekly_deaths_per_million" - - dimensions: - interval: biweekly - metric: change - indicators: - y: "{definitions.table}#biweekly_pct_growth_deaths" + # Views will be filled out programmatically. + [] + ``` -The `dimensions` field specifies selectors, and the `views` field defines views for the selection. Since there are numerous possible configurations, `views` are usually generated programmatically. However, it's a good idea to create a few of them manually to start. +The `dimensions` field specifies selectors, and the `views` field defines views for the selection. Since there are numerous possible configurations, `views` are usually generated programmatically (using function `etl.multidim.generate_views_for_dimensions`). You can also combine manually defined views with generated ones. See the `etl.multidim` module for available helper functions or refer to examples from `etl/steps/export/multidim/`. Feel free to add or modify the helper functions as needed. -The export step loads the YAML file, adds `views` to the config, and then calls the function. +The export step loads the data dependencies and the config YAML file, adds `views` to the config, and then pushes the configuration to the database. -```python title="etl/steps/export/multidim/covid/latest/covid.py" +```python title="etl/steps/export/multidim/energy/latest/energy_prices.py" def run(dest_dir: str) -> None: - engine = get_engine() - - # Load YAML file - config = paths.load_mdim_config("covid.deaths.yaml") + # + # Load inputs. + # + # Load data on energy prices. + ds_grapher = paths.load_dataset("energy_prices") + + # Read table of prices in euros. + tb_annual = ds_grapher.read("energy_prices_annual") + tb_monthly = ds_grapher.read("energy_prices_monthly") + + # + # Process data. + # + # Load configuration from adjacent yaml file. + config = paths.load_mdim_config() + + # Create views. + config["views"] = multidim.generate_views_for_dimensions( + dimensions=config["dimensions"], + tables=[tb_annual, tb_monthly], + dimensions_order_in_slug=("frequency", "source", "unit"), + warn_on_missing_combinations=False, + additional_config={"chartTypes": ["LineChart"], "hasMapTab": True, "tab": "map"}, + ) + + # + # Save outputs. + # + multidim.upsert_multidim_data_page(slug="mdd-energy-prices", config=config, engine=get_engine()) - multidim.upsert_multidim_data_page("mdd-energy", config, engine) ``` To see the multi-dimensional indicator in Admin, run ```bash -etlr export://multidim/energy/latest/energy --export +etlr export://multidim/energy/latest/energy_prices --export ``` -and check out the preview at http://staging-site-my-branch/admin/grapher/mdd-name. +and check out the preview at: http://staging-site-my-branch/admin/grapher/mdd-energy-prices ## Exporting data to GitHub diff --git a/docs/guides/data-work/update-data.md b/docs/guides/data-work/update-data.md index 1d02b05924d..c0a66de4366 100644 --- a/docs/guides/data-work/update-data.md +++ b/docs/guides/data-work/update-data.md @@ -12,9 +12,9 @@ This guide explains the general workflow to update a dataset that already exists In a nutshell, these are the steps to follow: - Switch to `master` branch (`git switch master`), and ensure it's up-to-date (`git pull`). - - Create a new branch and a draft pull request (PR) with a staging server: + - Create a new branch (name is auto-generated) and a draft pull request (PR) with a staging server: ```bash - etl pr update-{short_name} -c data -t "{short_name}: update" + etl pr "{short_name}: update" data ``` - Use the ETL Dashboard to create new versions of the steps (this will duplicate the code of the old steps). - Execute the newly created snapshot scripts, if any. @@ -47,7 +47,7 @@ This guide assumes you have already a [working installation of `etl`](../../../g - **Create a draft PR and a temporary staging server** - Create a PR with the following command (replace `{short_name}` with the short name of the dataset, e.g. `temperature-anomaly`): ```bash - etl pr update-{short_name} -c data -t "{short_name}: update" + etl pr "{short_name}: update" data ``` This will create a new git branch in your local repository with an empty commit, which will be pushed to remote. It will also create a draft pull request in github, and a staging server. diff --git a/docs/guides/etl-tips.md b/docs/guides/etl-tips.md index f0853f786e9..c96fa33e03f 100644 --- a/docs/guides/etl-tips.md +++ b/docs/guides/etl-tips.md @@ -136,7 +136,6 @@ You can adapt the function to your needs, and perform different kinds of expansi ) ``` - ## Deprecate code Our codebase has lots of code. Some of it may no longer be maintained or used. To avoid confusion, it is a good practice to slowly deprecate code. This can be done by adding a deprecation warning to the code, and then removing it after a certain period of time: @@ -149,3 +148,23 @@ from deprecated import deprecated Make sure to point users to an alternative function or method that they can use instead. Please deprecate function with care, and make sure to check if the function is widely used, and communicate the deprecation to the team. + +## Add entity annotations to your dataset +Just add the field `display.entityAnnotationsMap` to the desired indicator. + +```yaml +display: + entityAnnotationsMap: |- + Spain: Some annotation + France: Another annotation +``` + +!!! note "Space is limited" + + The space for annotations in charts is limited. Please be mindful and keep the annotations short and to the point. 2-4 words is usually enough, ideally 2. + + +
+ OWID chart with annotations +
Example chart with entity annotations. Note that the space for annotations.
+
diff --git a/docs/guides/pull-requests.md b/docs/guides/pull-requests.md index 21bcc6f7fb4..e2152923f13 100644 --- a/docs/guides/pull-requests.md +++ b/docs/guides/pull-requests.md @@ -19,7 +19,7 @@ Once you've created a PR, the automation user [@owidbot](https://github.com/owid - data-diff: Changes introduced in the data compared to PRODUCTION.
- Chart Upgrader + Chart Upgrader
PR, and [comment by @owidbot](https://github.com/owid/etl/pull/3563#issuecomment-2485397175), as of 19th November 2024
diff --git a/docs/guides/sharing-external.md b/docs/guides/sharing-external.md index 5e30cd059cc..f93b47e4376 100644 --- a/docs/guides/sharing-external.md +++ b/docs/guides/sharing-external.md @@ -4,20 +4,28 @@ tags: --- ## Sharing work with external people -Sometimes it's useful to share our work with external people to get feedback before publishing it to the public. Staging servers can be made available to public by creating a branch with `-public` suffix. This will make the staging site available at **https://staging-site-my-branch.tail6e23.ts.net**. +Sometimes it's useful to share our work with external people to get feedback before publishing it to the public. All staging servers are by default available to the public on `https://.owid.pages.dev/`. + + ### Sharing explorers To share explorers with the public, follow these steps: -1. Create a branch wiht `-public` suffix (thus creating staging server). +1. Set `isPublished` to `true` in your explorer configuration. +2. Commit to trigger a deploy (could be empty commit with `--allow-empty`) +3. Share your explorer with public on e.g. https://.owid.pages.dev/explorers/my-explorer. + + + diff --git a/docs/guides/staging-servers.md b/docs/guides/staging-servers.md index fa0c03e641b..6f63e0748c2 100644 --- a/docs/guides/staging-servers.md +++ b/docs/guides/staging-servers.md @@ -9,10 +9,12 @@ Dedicated staging servers are automatically created from every ETL pull request. !!! note "PR staging servers URLs" - You can visit your PR staging server at `http://staging-site-`. Note that `` might differ from the exact branch name, for example `feature/123` will be `feature-123` (all symbols are changed to dashes, and the maximum length is of 50 characters). + You can visit your PR staging server at `http://staging-site-` or `https://.owid.pages.dev/`. Note that `` might differ from the exact branch name, for example `feature/123` will be `feature-123` (all symbols are changed to dashes, and the maximum length is of 50 characters). For more details, refer to the [python code](https://github.com/owid/etl/blob/master/apps/chart_sync/cli.py#L284) generating `` from the branch name. +OWID site on staging servers is **public** by default. If you want to keep the work private (e.g. for embargoed data), use `-private` suffix in the branch name. This will make it available only on `http://staging-site-`. + Once the PR is ready and data manager merges into master, ETL will deploy the changes and automatically run `chart-sync` that syncs approved charts to production. Then the staging server is stopped and destroyed after 3 days. diff --git a/docs/ignore/generate_dynamic_docs.py b/docs/ignore/generate_dynamic_docs.py index 7cdd20532f4..97e8935f234 100644 --- a/docs/ignore/generate_dynamic_docs.py +++ b/docs/ignore/generate_dynamic_docs.py @@ -1,4 +1,5 @@ """Generate documentation files dynamically""" + import mkdocs_gen_files from etl.docs import render_dataset, render_indicator, render_origin, render_table diff --git a/etl/config.py b/etl/config.py index 5706ce503b7..59edff4129c 100644 --- a/etl/config.py +++ b/etl/config.py @@ -6,6 +6,7 @@ The environment variables and settings here are for publishing options, they're only important for OWID staff. """ + import os import pwd import re diff --git a/etl/data_helpers/population.py b/etl/data_helpers/population.py index fe1b2e68e3e..4ade7ffa82a 100644 --- a/etl/data_helpers/population.py +++ b/etl/data_helpers/population.py @@ -1,4 +1,5 @@ """Tools to load population data.""" + from typing import Any, Dict, Optional import pandas as pd diff --git a/etl/explorer.py b/etl/explorer.py index a250811ff21..d4355fbee18 100644 --- a/etl/explorer.py +++ b/etl/explorer.py @@ -5,6 +5,7 @@ - Compare content and content_raw - Test it in Pablo's scripts """ + from copy import copy from io import StringIO from pathlib import Path diff --git a/etl/files.py b/etl/files.py index 03ccfe83d1b..e77cd0c50a8 100644 --- a/etl/files.py +++ b/etl/files.py @@ -165,15 +165,13 @@ def yaml_dump( strip_lines: bool = True, replace_confusing_ascii: bool = False, width: int = 120, -) -> str: - ... +) -> str: ... @overload def yaml_dump( d: Dict[str, Any], stream: TextIO, strip_lines: bool = True, replace_confusing_ascii: bool = False, width: int = 120 -) -> None: - ... +) -> None: ... def yaml_dump( diff --git a/etl/git_helpers.py b/etl/git_helpers.py index 8b687aab963..5109520905e 100644 --- a/etl/git_helpers.py +++ b/etl/git_helpers.py @@ -7,7 +7,9 @@ Helpers for working with Git in an ETL flow. """ +import time from dataclasses import dataclass +from functools import wraps from pathlib import Path from typing import Any, Dict, Optional, Union, cast @@ -134,13 +136,28 @@ class GitError(Exception): pass +def log_time(func): + @wraps(func) + def wrapper(*args, **kwargs): + log.info(f"{func.__name__}.start") + start_time = time.time() + result = func(*args, **kwargs) + end_time = time.time() + log.info(f"{func.__name__}.end", t=end_time - start_time) + return result + + return wrapper + + +@log_time def get_changed_files( current_branch: Optional[str] = None, base_branch: Optional[str] = None, repo_path: Union[Path, str] = BASE_DIR, only_committed: bool = False, ) -> Dict[str, Dict[str, str]]: - """Return files that are different between the current branch and the specified base branch.""" + """Return files that are different between the current branch and the specified base branch. This can + be really slow if the number of files is large.""" repo = Repo(repo_path) if current_branch is None: diff --git a/etl/grapher_helpers.py b/etl/grapher_helpers.py index 67bf0df7bd3..b1426a752b1 100644 --- a/etl/grapher_helpers.py +++ b/etl/grapher_helpers.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field, is_dataclass from functools import lru_cache from pathlib import Path -from typing import Any, Dict, Iterable, List, Literal, Optional, Set, cast +from typing import Any, Dict, Iterable, List, Literal, Optional, Set, Union, cast import jinja2 import numpy as np @@ -10,10 +10,9 @@ import pymysql import sqlalchemy import structlog -from jinja2 import Environment from owid import catalog from owid.catalog import warnings -from owid.catalog.utils import underscore +from owid.catalog.utils import dynamic_yaml_load, dynamic_yaml_to_dict, underscore from sqlalchemy import text from sqlalchemy.engine import Engine from sqlalchemy.orm import Session @@ -23,7 +22,7 @@ log = structlog.get_logger() -jinja_env = Environment( +jinja_env = jinja2.Environment( block_start_string="<%", block_end_string="%>", variable_start_string="<<", @@ -32,8 +31,17 @@ comment_end_string="#>", trim_blocks=True, lstrip_blocks=True, + undefined=jinja2.StrictUndefined, ) + +# Helper function to raise an error with << raise("uh oh...") >> +def raise_helper(msg): + raise Exception(msg) + + +jinja_env.globals["raise"] = raise_helper + # this might work too pd.api.types.is_integer_dtype(col) INT_TYPES = tuple( {f"{n}{b}{p}" for n in ("int", "Int", "uint", "UInt") for b in ("8", "16", "32", "64") for p in ("", "[pyarrow]")} @@ -204,19 +212,29 @@ def _cached_jinja_template(text: str) -> jinja2.environment.Template: return jinja_env.from_string(text) -def _expand_jinja_text(text: str, dim_dict: Dict[str, str]) -> str: +def _expand_jinja_text(text: str, dim_dict: Dict[str, str]) -> Union[str, bool]: if not _uses_jinja(text): return text try: - return _cached_jinja_template(text).render(dim_dict) + # NOTE: we're stripping the result to avoid trailing newlines + out = _cached_jinja_template(text).render(dim_dict).strip() + # Convert strings to booleans. Getting boolean directly from Jinja is not possible + if out in ("false", "False", "FALSE"): + return False + elif out in ("true", "True", "TRUE"): + return True + return out except jinja2.exceptions.TemplateSyntaxError as e: new_message = f"{e.message}\n\nDimensions:\n{dim_dict}\n\nTemplate:\n{text}\n" raise e.__class__(new_message, e.lineno, e.name, e.filename) from e + except jinja2.exceptions.UndefinedError as e: + new_message = f"{e.message}\n\nDimensions:\n{dim_dict}\n\nTemplate:\n{text}\n" + raise e.__class__(new_message) from e def _expand_jinja(obj: Any, dim_dict: Dict[str, str]) -> Any: - """Expand Jinja in all metadata fields.""" + """Expand Jinja in all metadata fields. This modifies the original object in place.""" if obj is None: return None elif isinstance(obj, str): @@ -233,6 +251,33 @@ def _expand_jinja(obj: Any, dim_dict: Dict[str, str]) -> Any: return obj +def render_yaml_file(path: Union[str, Path], dim_dict: Dict[str, str]) -> Dict[str, Any]: + """Load YAML file and render Jinja in all fields. Return a dictionary. + + Usage: + from etl import grapher_helpers as gh + from etl import paths + + tb = Dataset(paths.DATA_DIR / "garden/who/2024-07-30/ghe")['ghe'] + gh.render_variable_meta(tb.my_col.m, dim_dict={"sex": "male"}) + """ + meta = dynamic_yaml_to_dict(dynamic_yaml_load(path)) + return _expand_jinja(meta, dim_dict) + + +def render_variable_meta(meta: catalog.VariableMeta, dim_dict: Dict[str, str]) -> catalog.VariableMeta: + """Render Jinja in all fields of VariableMeta. Return a new VariableMeta object. + + Usage: + # Create a playground.ipynb next to YAML file and run this in notebook + from etl import grapher_helpers as gh + m = gh.render_yaml_file("ghe.meta.yml", dim_dict={"sex": "male"}) + m['tables']['ghe']['variables']['death_count'] + """ + # TODO: move this as a method to VariableMeta class + return _expand_jinja(meta.copy(), dim_dict) + + def _title_column_and_dimensions(title: str, dim_dict: Dict[str, Any]) -> str: """Create new title from column title and dimensions. For instance `Deaths`, ["age", "sex"], ["10-18", "male"] will be converted into diff --git a/etl/grapher_io.py b/etl/grapher_io.py index 497c5409a49..53becf2c020 100644 --- a/etl/grapher_io.py +++ b/etl/grapher_io.py @@ -7,6 +7,7 @@ - Code using db_conn (pymysql.Connection objects). We should instead use sessions, or engines (or OWIDEnv) """ + import concurrent.futures import io import warnings diff --git a/etl/grapher_model.py b/etl/grapher_model.py index b83ee60c50f..01d78d9c39a 100644 --- a/etl/grapher_model.py +++ b/etl/grapher_model.py @@ -13,6 +13,7 @@ It is often necessary to add `default=None` or `init=False` to make pyright happy. """ + import copy import io import json @@ -425,13 +426,15 @@ def load_chart_variables(self, session: Session) -> Dict[int, "Variable"]: rows = session.execute(stm).scalars().all() variables = {r.id: r for r in rows} + # NOTE: columnSlug must always exist in dimensions and in chart_dimensions, so there's + # no need to include columnSlug # add columnSlug if present - column_slug = self.config.get("map", {}).get("columnSlug") - if column_slug: - try: - variables[int(column_slug)] = Variable.load_variable(session, column_slug) - except NoResultFound: - raise ValueError(f"columnSlug variable {column_slug} for chart {self.id} not found") + # column_slug = self.config.get("map", {}).get("columnSlug") + # if column_slug: + # try: + # variables[int(column_slug)] = Variable.load_variable(session, column_slug) + # except NoResultFound: + # raise ValueError(f"columnSlug variable {column_slug} for chart {self.id} not found") return variables @@ -1269,15 +1272,13 @@ def load_variables(cls, session: Session, variables_id: List[int]) -> List["Vari @classmethod def from_id_or_path( cls, session: Session, id_or_path: str | int, columns: Optional[List[str]] = None - ) -> "Variable": - ... + ) -> "Variable": ... @overload @classmethod def from_id_or_path( cls, session: Session, id_or_path: List[str | int], columns: Optional[List[str]] = None - ) -> List["Variable"]: - ... + ) -> List["Variable"]: ... @classmethod def from_id_or_path( @@ -1321,15 +1322,15 @@ def from_id_or_path( @overload @classmethod - def from_catalog_path(cls, session: Session, catalog_path: str, columns: Optional[List[str]] = None) -> "Variable": - ... + def from_catalog_path( + cls, session: Session, catalog_path: str, columns: Optional[List[str]] = None + ) -> "Variable": ... @overload @classmethod def from_catalog_path( cls, session: Session, catalog_path: List[str], columns: Optional[List[str]] = None - ) -> List["Variable"]: - ... + ) -> List["Variable"]: ... @classmethod def from_catalog_path( @@ -1344,13 +1345,13 @@ def from_catalog_path( @overload @classmethod - def from_id(cls, session: Session, variable_id: int, columns: Optional[List[str]] = None) -> "Variable": - ... + def from_id(cls, session: Session, variable_id: int, columns: Optional[List[str]] = None) -> "Variable": ... @overload @classmethod - def from_id(cls, session: Session, variable_id: List[int], columns: Optional[List[str]] = None) -> List["Variable"]: - ... + def from_id( + cls, session: Session, variable_id: List[int], columns: Optional[List[str]] = None + ) -> List["Variable"]: ... @classmethod def from_id( @@ -1864,7 +1865,9 @@ def _remap_variable_ids(config: Union[List, Dict[str, Any], Any], remap_ids: Dic out[k] = remap_ids[int(v)] # columnSlug is actually a variable id, but stored as a string (it wasn't a great decision) elif k in ("columnSlug", "sortColumnSlug"): - out[k] = str(remap_ids[int(v)]) + # sometimes columnSlug stays in config, but is deleted from dimensions. Ignore it + if int(v) in remap_ids: + out[k] = str(remap_ids[int(v)]) # if new fields with variable ids are added, try to handle them and raise a warning elif isinstance(v, int) and v in remap_ids: log.warning("remap_variable_ids.new_field", field=k, value=v) diff --git a/etl/helpers.py b/etl/helpers.py index ecc8df917b5..9b47f5ba353 100644 --- a/etl/helpers.py +++ b/etl/helpers.py @@ -594,7 +594,7 @@ def _get_attributes_from_step_name(step_name: str) -> Dict[str, str]: if channel_type.startswith(("walden", "snapshot")): channel = channel_type namespace, version, short_name = path.split("/") - elif channel_type.startswith(("data",)): + elif channel_type.startswith(("data", "export")): channel, namespace, version, short_name = path.split("/") else: raise WrongStepName diff --git a/etl/indicator_upgrade/indicator_update.py b/etl/indicator_upgrade/indicator_update.py index 193982e8ce9..5b5e5b8687e 100644 --- a/etl/indicator_upgrade/indicator_update.py +++ b/etl/indicator_upgrade/indicator_update.py @@ -3,7 +3,6 @@ These functions are used when there are updates on variables. They are used in the chart revision process. """ - from copy import deepcopy from typing import Any, Dict, List, Set diff --git a/etl/multidim.py b/etl/multidim.py index 0931cb0f946..f22c68cc87a 100644 --- a/etl/multidim.py +++ b/etl/multidim.py @@ -1,14 +1,20 @@ import json +from itertools import product import pandas as pd import yaml from sqlalchemy.engine import Engine +from structlog import get_logger from apps.chart_sync.admin_api import AdminAPI from etl.config import OWID_ENV from etl.db import read_sql +from etl.grapher_io import trim_long_variable_name from etl.helpers import map_indicator_path_to_id +# Initialize logger. +log = get_logger() + def upsert_multidim_data_page(slug: str, config: dict, engine: Engine) -> None: validate_multidim_config(config, engine) @@ -162,3 +168,103 @@ def fetch_variables_from_table(table: str, engine: Engine) -> pd.DataFrame: df_dims = pd.DataFrame(dims, index=df.index) return df.join(df_dims) + + +def generate_views_for_dimensions( + dimensions, tables, dimensions_order_in_slug=None, additional_config=None, warn_on_missing_combinations=True +): + """Generate individual views for all possible combinations of dimensions in a list of flattened tables. + + Parameters + ---------- + dimensions : List[Dict[str, Any]] + Dimensions, as given in the configuration of the multidim step, e.g. + [ + {'slug': 'frequency', 'name': 'Frequency', 'choices': [{'slug': 'annual','name': 'Annual'}, {'slug': 'monthly', 'name': 'Monthly'}]}, + {'slug': 'source', 'name': 'Energy source', 'choices': [{'slug': 'electricity', 'name': 'Electricity'}, {'slug': 'gas', 'name': 'Gas'}]}, + ... + ] + tables : List[Table] + Tables whose indicator views will be generated. + dimensions_order_in_slug : Tuple[str], optional + Dimension names, as they appear in "dimensions", and in the order in which they are spelled out in indicator names. For example, if indicator names are, e.g. annual_electricity_euros, then dimensions_order_in_slug would be ("frequency", "source", "unit"). + additional_config : _type_, optional + Additional config fields to add to each view, e.g. + {"chartTypes": ["LineChart"], "hasMapTab": True, "tab": "map"} + warn_on_missing_combinations : bool, optional + True to warn if any combination of dimensions is not found among the indicators in the given tables. + + Returns + ------- + results : List[Dict[str, Any]] + Views configuration, e.g. + [ + {'dimensions': {'frequency': 'annual', 'source': 'electricity', 'unit': 'euro'}, 'indicators': {'y': 'grapher/energy/2024-11-20/energy_prices/energy_prices_annual#annual_electricity_household_total_price_including_taxes_euro'}, + {'dimensions': {'frequency': 'annual', 'source': 'electricity', 'unit': 'pps'}, 'indicators': {'y': 'grapher/energy/2024-11-20/energy_prices/energy_prices_annual#annual_electricity_household_total_price_including_taxes_pps'}, + ... + ] + + """ + # Extract all choices for each dimension as (slug, choice_slug) pairs. + choices = {dim["slug"]: [choice["slug"] for choice in dim["choices"]] for dim in dimensions} + dimension_slugs_in_config = set(choices.keys()) + + # Sanity check for dimensions_order_in_slug. + if dimensions_order_in_slug: + dimension_slugs_in_order = set(dimensions_order_in_slug) + + # Check if any slug in the order is missing from the config. + missing_slugs = dimension_slugs_in_order - dimension_slugs_in_config + if missing_slugs: + raise ValueError( + f"The following dimensions are in 'dimensions_order_in_slug' but not in the config: {missing_slugs}" + ) + + # Check if any slug in the config is missing from the order. + extra_slugs = dimension_slugs_in_config - dimension_slugs_in_order + if extra_slugs: + log.warning( + f"The following dimensions are in the config but not in 'dimensions_order_in_slug': {extra_slugs}" + ) + + # Reorder choices to match the specified order. + choices = {dim: choices[dim] for dim in dimensions_order_in_slug if dim in choices} + + # Generate all combinations of the choices. + all_combinations = list(product(*choices.values())) + + # Create the views. + results = [] + for combination in all_combinations: + # Map dimension slugs to the chosen values. + dimension_mapping = {dim_slug: choice for dim_slug, choice in zip(choices.keys(), combination)} + slug_combination = "_".join(combination) + + # Find relevant tables for the current combination. + relevant_table = [] + for table in tables: + if slug_combination in table: + relevant_table.append(table) + + # Handle missing or multiple table matches. + if len(relevant_table) == 0: + if warn_on_missing_combinations: + log.warning(f"Combination {slug_combination} not found in tables") + continue + elif len(relevant_table) > 1: + log.warning(f"Combination {slug_combination} found in multiple tables: {relevant_table}") + + # Construct the indicator path. + indicator_path = f"{relevant_table[0].metadata.dataset.uri}/{relevant_table[0].metadata.short_name}#{trim_long_variable_name(slug_combination)}" + indicators = { + "y": indicator_path, + } + # Append the combination to results. + results.append({"dimensions": dimension_mapping, "indicators": indicators}) + + if additional_config: + # Include additional fields in all results. + for result in results: + result.update({"config": additional_config}) + + return results diff --git a/etl/paths.py b/etl/paths.py index f5a52647586..f55d81d3c71 100644 --- a/etl/paths.py +++ b/etl/paths.py @@ -23,7 +23,6 @@ # Snapshots SNAPSHOTS_DIR = BASE_DIR / "snapshots" -SNAPSHOTS_DIR_ARCHIVE = BASE_DIR / "snapshots_archive" # ETL library ETL_DIR = BASE_DIR / "etl" @@ -32,7 +31,6 @@ STEPS_MEADOW_DIR = STEPS_DATA_DIR / "meadow" STEPS_GARDEN_DIR = STEPS_DATA_DIR / "garden" STEPS_GRAPHER_DIR = STEPS_DATA_DIR / "grapher" -STEP_DIR_ARCHIVE = STEP_DIR / "archive" # Apps APPS_DIR = BASE_DIR / "apps" diff --git a/etl/scripts/anomalies/detect_anomalies.py b/etl/scripts/anomalies/detect_anomalies.py index 72422855b15..961453d0a18 100644 --- a/etl/scripts/anomalies/detect_anomalies.py +++ b/etl/scripts/anomalies/detect_anomalies.py @@ -1,6 +1,5 @@ -"""Detect anomalies in a given grapher dataset. +"""Detect anomalies in a given grapher dataset.""" -""" from typing import Dict, List, Optional, Tuple import click diff --git a/etl/scripts/faostat/archive/migrate_to_new_metadata.py b/etl/scripts/faostat/archive/migrate_to_new_metadata.py index 9e5349c81fe..2bc6a54d48c 100644 --- a/etl/scripts/faostat/archive/migrate_to_new_metadata.py +++ b/etl/scripts/faostat/archive/migrate_to_new_metadata.py @@ -5,6 +5,7 @@ In future updates we will need to adapt the scripts or etl steps to be able to easily update metadata. """ + import argparse import json diff --git a/etl/scripts/faostat/shared.py b/etl/scripts/faostat/shared.py index b0c0c881f9d..1c148a89502 100644 --- a/etl/scripts/faostat/shared.py +++ b/etl/scripts/faostat/shared.py @@ -1,6 +1,4 @@ -"""Common definitions used by scripts to create new snapshots and data steps. - -""" +"""Common definitions used by scripts to create new snapshots and data steps.""" import datetime as dt from typing import Dict, List, Tuple diff --git a/etl/snapshot.py b/etl/snapshot.py index d853f869666..6c6a83ebdc2 100644 --- a/etl/snapshot.py +++ b/etl/snapshot.py @@ -4,7 +4,7 @@ import tempfile from dataclasses import dataclass from pathlib import Path -from typing import Any, Iterator, Optional, Union, cast +from typing import Any, Dict, Iterator, Optional, Union, cast import owid.catalog.processing as pr import pandas as pd @@ -60,11 +60,7 @@ def path(self) -> Path: @property def metadata_path(self) -> Path: """Path to metadata file.""" - archive_path = Path(f"{paths.SNAPSHOTS_DIR_ARCHIVE / self.uri}.dvc") - if archive_path.exists(): - return archive_path - else: - return Path(f"{paths.SNAPSHOTS_DIR / self.uri}.dvc") + return Path(f"{paths.SNAPSHOTS_DIR / self.uri}.dvc") def _download_dvc_file(self, md5: str) -> None: """Download file from remote to self.path.""" @@ -233,6 +229,29 @@ def read_rda(self, *args, **kwargs) -> Table: """Read R data .rda file into a Table and populate it with metadata.""" return pr.read_rda(self.path, *args, metadata=self.to_table_metadata(), origin=self.metadata.origin, **kwargs) + def read_rda_multiple(self, *args, **kwargs) -> Dict[str, Table]: + """Read R data .rda file into multiple Tables and populate it with metadata. + + RData objects can contain multiple dataframes. + + Read specific dataframes from an RData file: + + ```python + tables = snap.read_rda_multiple(["tname1", "tname2"]) + ``` + + If you don't provide any table names, all tables will be read: + + ```python + tables = snap.read_rda_multiple() + ``` + + where tables is a key-value dictionary, and keys are the names of the tables (same as table short_names too). + """ + return pr.read_rda_multiple( + self.path, *args, metadata=self.to_table_metadata(), origin=self.metadata.origin, **kwargs + ) + def read_fwf(self, *args, **kwargs) -> Table: """Read a table of fixed-width formatted lines with metadata.""" return pr.read_fwf(self.path, *args, metadata=self.to_table_metadata(), origin=self.metadata.origin, **kwargs) diff --git a/etl/steps/__init__.py b/etl/steps/__init__.py index 9be0e7c4132..a3d2ed5aa6b 100644 --- a/etl/steps/__init__.py +++ b/etl/steps/__init__.py @@ -117,7 +117,8 @@ def filter_to_subgraph( if exact_match: included = set(includes) & all_steps else: - included = {s for s in all_steps if any(re.findall(pattern, s) for pattern in includes)} + compiled_includes = [re.compile(p) for p in includes] + included = {s for s in all_steps if any(p.search(s) for p in compiled_includes)} if only: # Only include explicitly selected nodes @@ -369,14 +370,11 @@ class Step(Protocol): version: str dependencies: List["Step"] - def run(self) -> None: - ... + def run(self) -> None: ... - def is_dirty(self) -> bool: - ... + def is_dirty(self) -> bool: ... - def checksum_output(self) -> str: - ... + def checksum_output(self) -> str: ... def __str__(self) -> str: raise NotImplementedError() diff --git a/etl/steps/data/explorers/wb/latest/world_bank_pip.py b/etl/steps/data/explorers/wb/latest/world_bank_pip.py index 0fbb35a7149..b2db098ae4b 100644 --- a/etl/steps/data/explorers/wb/latest/world_bank_pip.py +++ b/etl/steps/data/explorers/wb/latest/world_bank_pip.py @@ -5,7 +5,6 @@ """ - from owid.catalog import Dataset, Table from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/external/co2_data/latest/owid_co2.py b/etl/steps/data/external/co2_data/latest/owid_co2.py index 659f7ec50f9..1d30f5f9556 100644 --- a/etl/steps/data/external/co2_data/latest/owid_co2.py +++ b/etl/steps/data/external/co2_data/latest/owid_co2.py @@ -11,7 +11,6 @@ """ - import numpy as np from owid.catalog import Dataset, Origin, Table diff --git a/etl/steps/data/garden/antibiotics/2024-10-09/gram_level.py b/etl/steps/data/garden/antibiotics/2024-10-09/gram_level.py index fdedd02d373..f49d46fe58e 100644 --- a/etl/steps/data/garden/antibiotics/2024-10-09/gram_level.py +++ b/etl/steps/data/garden/antibiotics/2024-10-09/gram_level.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + from owid.catalog import Dataset, Table from owid.catalog import processing as pr diff --git a/etl/steps/data/garden/antibiotics/2024-11-12/antimicrobial_usage.meta.yml b/etl/steps/data/garden/antibiotics/2024-11-12/antimicrobial_usage.meta.yml index 0a97108591e..0dae03f5732 100644 --- a/etl/steps/data/garden/antibiotics/2024-11-12/antimicrobial_usage.meta.yml +++ b/etl/steps/data/garden/antibiotics/2024-11-12/antimicrobial_usage.meta.yml @@ -4,18 +4,16 @@ definitions: presentation: topic_tags: - Global Health - aware_description: - <% if aware == "A" %> + aware_description: <% if awarelabel == "Access" %> Access antibiotics have activity against a wide range of common pathogens and show lower resistance potential than antibiotics in the other groups. - <% elif aware == "W" %> + <% elif awarelabel == "Watch" %> Watch antibiotic have higher resistance potential and include most of the highest priority agents among the Critically Important Antimicrobials for Human Medicine and/or antibiotics that are at relatively high risk of bacterial resistance. - <% elif aware == "R" %> + <% elif awarelabel == "Reserve" %> Reserve antibiotics should be reserved for treatment of confirmed or suspected infections due to multi-drug-resistant organisms. Reserve group antibiotics should be treated as “last resort” options. - <% elif aware == "O" %> + <% elif awarelabel == "Not classified/recommended" %> The use of the Not classified/Not recommended antibiotics is not evidence-based, nor recommended in high-quality international guidelines. WHO does not recommend the use of these antibiotics in clinical practice. <% endif %> - routeofadministration: - <% if routeofadministration == "O" %> + routeofadministration: <% if routeofadministration == "O" %> orally administered <% elif routeofadministration == "P" %> parentearally administered @@ -24,36 +22,112 @@ definitions: <% elif routeofadministration == "I" %> inhaled <% endif %> - + antimicrobialclass: + <% if antimicrobialclass == "Antibacterials (ATC J01, A07AA, P01AB, ATC J04A)" %> + antibiotics including antituberculosis drugs + <% elif antimicrobialclass == "Antimalarials (ATC P01B)" %> + antimalarials + <% elif antimicrobialclass == "Antimycotics and antifungals for systemic use (J02, D01B)" %> + antifungals + <% elif antimicrobialclass == "Antivirals for systemic use (ATC J05)" %> + antivirals + <% elif antimicrobialclass == "Drugs for the treatment of tuberculosis (ATC J04A)" %> + antituberculosis drugs + <% endif %> # Learn more about the available fields: # http://docs.owid.io/projects/etl/architecture/metadata/reference/ dataset: - update_period_days: 308 - + update_period_days: 365 tables: class: variables: ddd: - title: Defined daily doses of {definitions.routeofadministration} << antimicrobialclass>> - << atc4name.lower() >> used - description_short: Volume of antimicrobials used in a given year. - #description_processing: <> + title: Defined daily doses of {definitions.routeofadministration} << antimicrobialclass.lower()>> - << atc4name.lower() >> used + description_short: Total [defined daily doses](#dod:defined-daily-doses) of antimicrobials used in a given year. unit: defined daily doses + display: + numDecimalPlaces: 0 did: title: Defined daily doses per 1000 inhabitants per day of {definitions.routeofadministration} << antimicrobialclass>> - << atc4name.lower() >> used - description_short: Volume of antimicrobials used per 1000 inhabitants per day. - #description_processing: <> + description_short: Total [defined daily doses](#dod:defined-daily-doses) of antimicrobials used per 1000 inhabitants per day. unit: defined daily doses per 1000 inhabitants per day + display: + numDecimalPlaces: 1 + class_aggregated: + variables: + ddd_anti_malarials: + title: Defined daily doses of antimalarials used + description_short: Total [defined daily doses](#dod:defined-daily-doses) of antimalarials used in a given year. + unit: defined daily doses + display: + numDecimalPlaces: 0 + ddd_antibacterials_and_antituberculosis: + title: Defined daily doses of antibiotics and antituberculosis drugs used + description_short: Total [defined daily doses](#dod:defined-daily-doses) of antibiotics and antituberculosis drugs used in a given year. + unit: defined daily doses + display: + numDecimalPlaces: 0 + ddd_antifungals: + title: Defined daily doses of antifungals used + description_short: Total [defined daily doses](#dod:defined-daily-doses) of antifungals used in a given year. + unit: defined daily doses + display: + numDecimalPlaces: 0 + ddd_antituberculosis: + title: Defined daily doses of antituberculosis drugs used + description_short: Total [defined daily doses](#dod:defined-daily-doses) of antituberculosis drugs used in a given year. + unit: defined daily doses + display: + numDecimalPlaces: 0 + ddd_antivirals: + title: Defined daily doses of antivirals used + description_short: Total [defined daily doses](#dod:defined-daily-doses) of antivirals used in a given year. + unit: defined daily doses + display: + numDecimalPlaces: 0 + did_anti_malarials: + title: Defined daily doses of antimalarials used per 1,000 inhabitants per day + description_short: Total [defined daily doses](#dod:defined-daily-doses) of antimalarials used in a given year per 1,000 inhabitants per day. + unit: defined daily doses per 1,000 inhabitants per day + display: + numDecimalPlaces: 1 + did_antibacterials_and_antituberculosis: + title: Defined daily doses of antibiotics and antituberculosis drugs used per 1,000 inhabitants per day + description_short: Total [defined daily doses](#dod:defined-daily-doses) of antibiotics and antituberculosis drugs used in a given year per 1,000 inhabitants per day. + unit: defined daily doses per 1,000 inhabitants per day + display: + numDecimalPlaces: 1 + did_antifungals: + title: Defined daily doses of antifungals used per 1,000 inhabitants per day + description_short: Total [defined daily doses](#dod:defined-daily-doses) of antifungals used in a given year per 1,000 inhabitants per day. + unit: defined daily doses per 1,000 inhabitants per day + display: + numDecimalPlaces: 1 + did_antituberculosis: + title: Defined daily doses of antituberculosis drugs used per 1,000 inhabitants per day + description_short: Total [defined daily doses](#dod:defined-daily-doses) of antituberculosis drugs used in a given year per 1,000 inhabitants per day. + unit: defined daily doses per 1,000 inhabitants per day + display: + numDecimalPlaces: 1 + did_antivirals: + title: Defined daily doses of antivirals used per 1,000 inhabitants per day + description_short: Total [defined daily doses](#dod:defined-daily-doses) of antivirals used in a given year per 1,000 inhabitants per day. + unit: defined daily doses per 1,000 inhabitants per day + display: + numDecimalPlaces: 1 aware: variables: ddd: title: Defined daily doses of << awarelabel >> antibiotics used - description_short: "Volume of AWaRe category: << awarelabel >> antibiotics used in a given year. {definitions.aware_description}" - #description_processing: <> + description_short: "Total [defined daily doses](#dod:defined-daily-doses) of AWaRe category: << awarelabel >> antibiotics used in a given year. {definitions.aware_description}" unit: defined daily doses + display: + numDecimalPlaces: 0 did: title: Defined daily doses per 1000 inhabitants per day of << awarelabel>> antibiotics used - description_short: "Volume of AWaRe category: <> used per 1000 inhabitants per day. {definitions.aware_description}" - #description_processing: <> + description_short: "Total [defined daily doses](#dod:defined-daily-doses) of AWaRe category: <> used per 1000 inhabitants per day. {definitions.aware_description}" unit: defined daily doses per 1000 inhabitants per day + display: + numDecimalPlaces: 1 diff --git a/etl/steps/data/garden/antibiotics/2024-11-12/antimicrobial_usage.py b/etl/steps/data/garden/antibiotics/2024-11-12/antimicrobial_usage.py index d167c0d1737..2191393054b 100644 --- a/etl/steps/data/garden/antibiotics/2024-11-12/antimicrobial_usage.py +++ b/etl/steps/data/garden/antibiotics/2024-11-12/antimicrobial_usage.py @@ -1,5 +1,8 @@ """Load a meadow dataset and create a garden dataset.""" +from owid.catalog import Table +from owid.catalog import processing as pr + from etl.data_helpers import geo from etl.helpers import PathFinder, create_dataset @@ -23,6 +26,11 @@ def run(dest_dir: str) -> None: tb_class = geo.harmonize_countries(df=tb_class, countries_file=paths.country_mapping_path) tb_aware = geo.harmonize_countries(df=tb_aware, countries_file=paths.country_mapping_path) + # Tidy notes column + tb_class = tidy_notes(tb_class) + # Aggregate by antimicrobial class + tb_class_agg, tb_notes = aggregate_antimicrobial_classes(tb_class) + # Save the origins of the aggregated table to insert back in later # Drop columns that are not needed in the garden dataset. tb_class = tb_class.drop( columns=["whoregioncode", "whoregionname", "countryiso3", "incomeworldbankjune", "atc4", "notes"] @@ -31,14 +39,151 @@ def run(dest_dir: str) -> None: tb_class = tb_class.format(["country", "year", "antimicrobialclass", "atc4name", "routeofadministration"]) tb_aware = tb_aware.format(["country", "year", "awarelabel"]) + tb_class_agg = pivot_aggregated_table(tb_class_agg, tb_notes) + tb_class_agg = tb_class_agg.format(["country", "year"], short_name="class_aggregated") # # Save outputs. # # Create a new garden dataset with the same metadata as the meadow dataset. ds_garden = create_dataset( - dest_dir, tables=[tb_class, tb_aware], check_variables_metadata=True, default_metadata=ds_meadow.metadata + dest_dir, + tables=[tb_class, tb_aware, tb_class_agg], + check_variables_metadata=True, + default_metadata=ds_meadow.metadata, ) # Save changes in the new garden dataset. ds_garden.save() + + +def pivot_aggregated_table(tb_class_agg: Table, tb_notes: Table) -> Table: + """ + Pivot the aggregated table to have a column for each antimicrobial class, then add the description_processing metadata + """ + + tb_notes_dict = { + "Antibacterials (ATC J01, A07AA, P01AB)": "antibacterials", + "Antimalarials (ATC P01B)": "anti_malarials", + "Antimycotics and antifungals for systemic use (J02, D01B)": "antifungals", + "Antivirals for systemic use (ATC J05)": "antivirals", + "Drugs for the treatment of tuberculosis (ATC J04A)": "antituberculosis", + "Antibacterials (ATC J01, A07AA, P01AB, ATC J04A)": "antibacterials_and_antituberculosis", + } + tb_notes["category"] = tb_notes["antimicrobialclass"].map(tb_notes_dict) + tb_class_agg = tb_class_agg.copy(deep=True) + tb_class_agg["antimicrobialclass"] = tb_class_agg["antimicrobialclass"].replace(tb_notes_dict) + tb_class_agg = tb_class_agg.pivot( + index=["country", "year"], columns="antimicrobialclass", values=["ddd", "did"], join_column_levels_with="_" + ) + tb_class_agg = tb_class_agg.reset_index(drop=True) + + for key in tb_notes_dict.values(): + if f"ddd_{key}" in tb_class_agg.columns: + tb_class_agg[f"ddd_{key}"].metadata.description_key = tb_notes["description_processing"][ + tb_notes["category"] == key + ] + if f"did_{key}" in tb_class_agg.columns: + tb_class_agg[f"did_{key}"].metadata.description_key = tb_notes["description_processing"][ + tb_notes["category"] == key + ] + return tb_class_agg + + +def aggregate_antimicrobial_classes(tb: Table) -> Table: + """ + Aggregating by antimicrobial class, we want to combine antibacterials and antituberculosis, but also keep antituberculosis separately + """ + tb = tb.copy(deep=True) + # Convert the column to strings (if not already done) + tb["antimicrobialclass"] = tb["antimicrobialclass"].astype("string") + + # Create a completely independent copy of antituberculosis rows and reset its index + msk = tb["antimicrobialclass"] == "Drugs for the treatment of tuberculosis (ATC J04A)" + tb_anti_tb = tb[msk].reset_index(drop=True) + assert len(tb_anti_tb["antimicrobialclass"].unique()) == 1 + + # Modify antimicrobialclass in tb + tb["antimicrobialclass"] = tb["antimicrobialclass"].replace( + { + "Drugs for the treatment of tuberculosis (ATC J04A)": "Antibacterials (ATC J01, A07AA, P01AB, ATC J04A)", + "Antibacterials (ATC J01, A07AA, P01AB)": "Antibacterials (ATC J01, A07AA, P01AB, ATC J04A)", + }, + ) + expected_class_values = { + "Antibacterials (ATC J01, A07AA, P01AB, ATC J04A)", + "Antimalarials (ATC P01B)", + "Antimycotics and antifungals for systemic use (J02, D01B)", + "Antivirals for systemic use (ATC J05)", + } + actual_values = set(tb["antimicrobialclass"].unique()) + assert actual_values == expected_class_values + # Format the notes tables before it's removed + tb_notes = tb[["country", "year", "antimicrobialclass", "notes"]].dropna(subset=["notes"]) + tb_notes = format_notes(tb_notes) + + # Aggregate the data + tb = tb.groupby(["country", "year", "antimicrobialclass"], dropna=False)[["ddd", "did"]].sum().reset_index() + assert len(tb["antimicrobialclass"].unique()) == 4 + # Add the antituberculosis data back to tb + tb_anti_tb = ( + tb_anti_tb.groupby(["country", "year", "antimicrobialclass"], dropna=False)[["ddd", "did"]].sum().reset_index() + ) + tb_combined = pr.concat([tb, tb_anti_tb]) + + tb_combined.set_index(["country", "year", "antimicrobialclass"], verify_integrity=True) + + return tb_combined, tb_notes + + +def format_notes(tb_notes: Table) -> Table: + """ + Format notes column + """ + for note in tb_notes["notes"].unique(): + msk = tb_notes["notes"] == note + tb_note = tb_notes[msk] + countries = tb_note["country"].unique() + countries_formatted = combine_countries(countries) + description_processing_string = f"For {countries_formatted}: {note}" + tb_notes.loc[msk, "description_processing"] = description_processing_string + # Creating onedescription processing for each antimicrobial class, the variable unit + tb_desc = ( + tb_notes.dropna(subset=["description_processing"]) # Remove NaNs + .groupby(["antimicrobialclass"])["description_processing"] + .apply(lambda x: list(set(x))) # Combine unique values into a list + .reset_index() + ) + + return tb_desc + + +def tidy_notes(tb_class: Table) -> Table: + """ + Tidy notes column - improve the syntax and fix spelling errors + """ + notes_dict = { + "Only consumption in the community reported": "only antimicrobial consumption in the community is reported.", + "For antimycotics and antifungals: only J02 reported": "for antimycotics and antifungals, only antimycotics for systemic use (ATC code J02) are reported.", + "For antibiotics: only J01 and P01AB reported": "for antibiotics, only antibiotics for systemic use (ATC code J01) and nitroimidazole derivatives (ATC code P01AB) are reported.", + "For antibiotics: only J01 reported": "for antibiotics, only antibiotics for systemic use (ATC code J01) are reported", + "For antifungals: only use in the hospital reported": "for antifungals, only those used in hospitals are reported.", + "Data incomplete since not collected from all sources of data": "data is incomplete since it's not collected from all sources.", + "Only consumption in the public sector reported and this is estimated to reppresent less than 90% of the antimicrobial used in the country ": "only consumption in the public sector reported and this is estimated to represent less than 90% of total antimicrobial usage.", + "Data incomplete: not all antibiotics reported systematically": "data is incomplete, not all antibiotics reported systematically.", + "For antituberculosis medicines: data are incomplete": "data are incomplete for antituberculosis medicines.", + } + tb_class["notes"] = tb_class["notes"].replace(notes_dict) + return tb_class + + +def combine_countries(countries): + # Combine countries into a string + if not countries: + return "" + elif len(countries) == 1: + return countries[0] + elif len(countries) == 2: + return " and ".join(countries) + else: + return ", ".join(countries[:-1]) + " and " + countries[-1] diff --git a/etl/steps/data/garden/antibiotics/2024-11-15/testing_coverage.py b/etl/steps/data/garden/antibiotics/2024-11-15/testing_coverage.py index 379ae3e3c4b..591d82b7d91 100644 --- a/etl/steps/data/garden/antibiotics/2024-11-15/testing_coverage.py +++ b/etl/steps/data/garden/antibiotics/2024-11-15/testing_coverage.py @@ -11,7 +11,7 @@ WHO_REGION_MEMBERS = { "African Region (WHO)": 47, "World": 194, - "Eastern Mediterranean (WHO)": 21, + "Eastern Mediterranean (WHO)": 22, "European Region (WHO)": 53, "Region of the Americas (WHO)": 35, "South-East Asia Region (WHO)": 11, diff --git a/etl/steps/data/garden/antibiotics/2024-11-20/bloodstream_amr.meta.yml b/etl/steps/data/garden/antibiotics/2024-11-20/bloodstream_amr.meta.yml deleted file mode 100644 index 5ae8c20ec36..00000000000 --- a/etl/steps/data/garden/antibiotics/2024-11-20/bloodstream_amr.meta.yml +++ /dev/null @@ -1,74 +0,0 @@ -definitions: - common: - presentation: - topic_tags: - - Antibiotics - display: - numSignificantFigures: 3 - - pathogen_type: - <%- if pathogen_type == "Fungi" -%> - fungus - <%- elif pathogen_type == "Viruses" -%> - virus - <%- else -%> - << pathogen_type.lower() >> - <%- endif -%> - counterfactual: - <% if counterfactual == "Associated" %> - where the death was associated with antibiotic resistance - <% elif counterfactual == "Susceptible" %> - where the bacteria causing the death were susceptible to antibiotics - <% elif counterfactual == "Untested" %> - where the pathogen was not tested for antibiotic resistance - <% endif %> - - - -# Learn more about the available fields: -# http://docs.owid.io/projects/etl/architecture/metadata/reference/ -dataset: - update_period_days: 365 - - -tables: - bloodstream_amr: - variables: - value: - title: Neonatal deaths from << pathogen >> infections {definitions.counterfactual} - unit: deaths - description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from << pathogen >> infections, {definitions.counterfactual} << pathogen >> is a {definitions.pathogen_type}. - display: - name: << pathogen >> - << counterfactual >> - upper: - title: Upper bound of neonatal deaths from << pathogen >> infections {definitions.counterfactual} - unit: deaths - description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from << pathogen >> infections, {definitions.counterfactual}. << pathogen >> is a {definitions.pathogen_type}. - display: - name: << pathogen >> - << counterfactual >> - lower: - title: Lower bound of neonatal deaths from << pathogen >> infections {definitions.counterfactual} - unit: deaths - description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from << pathogen >> infections, {definitions.counterfactual}. << pathogen >> is a {definitions.pathogen_type}. - display: - name: << pathogen >> - << counterfactual >> - amr_entity: - variables: - value: - title: Global neonatal deaths from bloodstream infections, by pathogen {definitions.counterfactual} - unit: deaths - description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from bloodstream infections, {definitions.counterfactual}. - display: - name: << counterfactual >> - upper: - title: Upper bound of global neonatal deaths from bloodstream infections, by pathogen {definitions.counterfactual} - unit: deaths - description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from bloodstream infections, {definitions.counterfactual}. - display: - name: << counterfactual >> - lower: - title: Lower bound of global neonatal deaths from bloodstream infections, by pathogen {definitions.counterfactual} - unit: deaths - description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from bloodstream infections, {definitions.counterfactual}. - display: - name: << counterfactual >> \ No newline at end of file diff --git a/etl/steps/data/garden/antibiotics/2024-11-20/microbe.meta.yml b/etl/steps/data/garden/antibiotics/2024-11-20/microbe.meta.yml index e20d3c35f3d..7a32139565a 100644 --- a/etl/steps/data/garden/antibiotics/2024-11-20/microbe.meta.yml +++ b/etl/steps/data/garden/antibiotics/2024-11-20/microbe.meta.yml @@ -6,8 +6,7 @@ definitions: - Global Health display: numSignificantFigures: 3 - pathogen_type: - <%- if pathogen_type == "Fungi" -%> + pathogen_type: <%- if pathogen_type == "Fungi" -%> fungus <%- elif pathogen_type == "Viruses" -%> virus @@ -15,13 +14,11 @@ definitions: << pathogen_type.lower() >> <%- endif -%> - # Learn more about the available fields: # http://docs.owid.io/projects/etl/architecture/metadata/reference/ dataset: update_period_days: 365 - tables: microbe: variables: @@ -29,14 +26,26 @@ tables: title: Neonatal deaths from << pathogen >> infections unit: deaths description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from << pathogen >> infections. << pathogen >> is a {definitions.pathogen_type}. + display: + roundingMode: significantFigures + numSignificantFigures: 3 + name: << pathogen >> upper: title: Upper bound of neonatal deaths from << pathogen >> infections unit: deaths description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from << pathogen >> infections. << pathogen >> is a {definitions.pathogen_type}. + display: + roundingMode: significantFigures + numSignificantFigures: 3 + name: << pathogen >> lower: title: Lower bound of neonatal deaths from << pathogen >> infections unit: deaths description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from << pathogen >> infections. << pathogen >> is a {definitions.pathogen_type}. + display: + roundingMode: significantFigures + numSignificantFigures: 3 + name: << pathogen >> pathogen_entity: variables: value: @@ -50,4 +59,4 @@ tables: lower: title: Lower bound of global neonatal deaths from bloodstream infections, by pathogen unit: deaths - description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from bloodstream infections. \ No newline at end of file + description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from bloodstream infections. diff --git a/etl/steps/data/garden/antibiotics/2024-11-20/bloodstream_amr.countries.json b/etl/steps/data/garden/antibiotics/2024-12-02/microbe_amr.countries.json similarity index 100% rename from etl/steps/data/garden/antibiotics/2024-11-20/bloodstream_amr.countries.json rename to etl/steps/data/garden/antibiotics/2024-12-02/microbe_amr.countries.json diff --git a/etl/steps/data/garden/antibiotics/2024-12-02/microbe_amr.meta.yml b/etl/steps/data/garden/antibiotics/2024-12-02/microbe_amr.meta.yml new file mode 100644 index 00000000000..0d28fc93f14 --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-02/microbe_amr.meta.yml @@ -0,0 +1,44 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Antibiotics + description_key: + - The pathogen and antimicrobial pairs that were tested for resistance can be found on pages 51-52 in the [Supplementary Materials (part 1)](#https://www.thelancet.com/cms/10.1016/S0140-6736(24)01867-1/attachment/005e2559-204f-46d7-b2e3-27b2091209aa/mmc1.pdf) of the original publication. + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + update_period_days: 365 + +tables: + microbe_amr: + variables: + amr_attributable_deaths: + title: Global neonatal deaths from infections attributed to AMR, by syndrome + unit: deaths + description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from infections that are attributed to antimicrobial resistance. + presentation: + title_public: Global neonatal deaths from infections attributed to AMR + display: + roundingMode: significantFigures + numSignificantFigures: 3 + non_amr_attributable_deaths: + title: Global neonatal deaths from infections not attributed to AMR, by syndrome + unit: deaths + description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from infections not attributed to antimicrobial resistance. + presentation: + title_public: Global neonatal deaths from infections not attributed to AMR + display: + roundingMode: significantFigures + numSignificantFigures: 3 + total_deaths: + title: Total global neonatal deaths from infections + unit: deaths + description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from infections. + presentation: + title_public: Total global neonatal deaths from infections + display: + roundingMode: significantFigures + numSignificantFigures: 3 diff --git a/etl/steps/data/garden/antibiotics/2024-12-02/microbe_amr.py b/etl/steps/data/garden/antibiotics/2024-12-02/microbe_amr.py new file mode 100644 index 00000000000..62db655bce1 --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-02/microbe_amr.py @@ -0,0 +1,98 @@ +"""Load a meadow dataset and create a garden dataset.""" + +from owid.catalog import Table + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("microbe_amr") + ds_total = paths.load_dataset("total_syndrome") + # Read table from meadow dataset. + tb = ds_meadow.read("microbe_amr") + tb_total = ds_total.read("total_syndrome") + # + # Process data. + # + tb = geo.harmonize_countries(df=tb, countries_file=paths.country_mapping_path) + tb_total = geo.harmonize_countries(df=tb_total, countries_file=paths.country_mapping_path) + + # We want three variables, total, amr attributable and amr non-attributable + + tb_amr = ( + tb[tb["counterfactual"] == "Attributable"] + .rename(columns={"value": "amr_attributable_deaths"}, errors="raise") + .drop(columns=["lower", "upper"]) + ) + tb_total = tb_total.rename(columns={"value": "total_deaths"}, errors="raise").drop(columns=["lower", "upper"]) + + tb = tb_amr.merge(tb_total, on=["country", "year", "infectious_syndrome"], how="inner") + tb["non_amr_attributable_deaths"] = tb["total_deaths"] - tb["amr_attributable_deaths"] + # Rename syndromes to be shorter for use in stacked bar charts + tb = rename_syndromes(tb) + + # Reformatting the data so it can be used in stacked bar charts + tb = ( + tb.drop(columns=["country"]).rename(columns={"infectious_syndrome": "country"}).drop(columns=["counterfactual"]) + ) + + tb = tb.format(["country", "year"]) + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata + ) + + # Save changes in the new garden dataset. + ds_garden.save() + + +def rename_syndromes(tb: Table) -> Table: + """ + Rename syndromes to be shorter for use in stacked bar charts. + Ensure all infectious syndromes are replaced. + """ + name_dict = { + "Bloodstream infections": "Bloodstream infections", + "Lower respiratory infections": "Lower respiratory infections", + "Diarrhea": "Diarrhea", + "Meningitis": "Meningitis", + "Infections of the skin and subcutaneous systems": "Skin infections", + "Urinary tract infections and pyelonephritis": "Kidney and urinary tract infections", + "Peritoneal and intra-abdominal infections": "Abdominal infections", + "Tuberculosis": "Tuberculosis", + "Endocarditis": "Endocarditis", + "Typhoid fever, paratyphoid fever, and invasive non-typhoidal Salmonella": "Typhoid, paratyphoid, and iNTS", + "Infections of bones, joints, and related organs": "Bone and joint infections", + "Other unspecified site infections": "Other infections", + "Other parasitic infections": "Other parasitic infections", + "Oral infections": "Oral infections", + "Myelitis, meningoencephalitis, and other central nervous system infections": "Central nervous system infections", + "Upper respiratory infections": "Upper respiratory infections", + "Hepatitis": "Hepatitis", + "Eye infections": "Eye infections", + "Encephalitis": "Encephalitis", + "Carditis, myocarditis, and pericarditis": "Heart inflammation", + "Sexually transmitted infections": "Sexually transmitted infections", + } + + # Find unmatched syndromes + unmatched_syndromes = set(tb["infectious_syndrome"].unique()) - set(name_dict.keys()) + if unmatched_syndromes: + raise ValueError(f"The following syndromes were not found in the name dictionary: {unmatched_syndromes}") + + # Replace syndromes + tb["infectious_syndrome"] = tb["infectious_syndrome"].replace(name_dict) + + return tb diff --git a/etl/steps/data/garden/antibiotics/2024-11-24/total_syndrome.countries.json b/etl/steps/data/garden/antibiotics/2024-12-02/microbe_neonatal_amr.countries.json similarity index 100% rename from etl/steps/data/garden/antibiotics/2024-11-24/total_syndrome.countries.json rename to etl/steps/data/garden/antibiotics/2024-12-02/microbe_neonatal_amr.countries.json diff --git a/etl/steps/data/garden/antibiotics/2024-12-02/microbe_neonatal_amr.meta.yml b/etl/steps/data/garden/antibiotics/2024-12-02/microbe_neonatal_amr.meta.yml new file mode 100644 index 00000000000..20fca81f6b3 --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-02/microbe_neonatal_amr.meta.yml @@ -0,0 +1,42 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Antibiotics + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + update_period_days: 365 + +tables: + microbe_neonatal_amr: + variables: + amr_attributable_deaths: + title: Global neonatal deaths from bloodstream infections attributed to AMR, by pathogen + unit: deaths + description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from bloodstream infections that are attributed to antimicrobial resistance. + presentation: + title_public: Global neonatal deaths from bloodstream infections attributed to AMR + display: + roundingMode: significantFigures + numSignificantFigures: 3 + non_amr_attributable_deaths: + title: Global neonatal deaths from bloodstream infections not attributed to AMR, by pathogen + unit: deaths + description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from bloodstream infections not attributed to antimicrobial resistance. + presentation: + title_public: Global neonatal deaths from bloodstream infections not attributed to AMR + display: + roundingMode: significantFigures + numSignificantFigures: 3 + total_deaths: + title: Total global neonatal deaths from bloodstream infections + unit: deaths + description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from bloodstream infections. + presentation: + title_public: Total global neonatal deaths from bloodstream infections + display: + roundingMode: significantFigures + numSignificantFigures: 3 diff --git a/etl/steps/data/garden/antibiotics/2024-12-02/microbe_neonatal_amr.py b/etl/steps/data/garden/antibiotics/2024-12-02/microbe_neonatal_amr.py new file mode 100644 index 00000000000..8d09ad6b51b --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-02/microbe_neonatal_amr.py @@ -0,0 +1,60 @@ +"""Load a meadow dataset and create a garden dataset.""" + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("microbe_neonatal_amr") + ds_total = paths.load_dataset("microbe") + # Read table from meadow dataset. + tb = ds_meadow.read("microbe_neonatal_amr") + tb_total = ds_total.read("microbe") + # + # Process data. + # + # We want three variables, total, amr attributable and amr non-attributable + + tb_amr = tb.rename(columns={"value": "amr_attributable_deaths"}, errors="raise").drop(columns=["lower", "upper"]) + tb_total = tb_total.rename(columns={"value": "total_deaths"}, errors="raise").drop( + columns=[ + "lower", + "upper", + "age", + "sex", + "measure", + "metric", + "pathogen_type", + "infectious_syndrome", + "counterfactual", + ] + ) + tb_total = tb_total[tb_total["year"] == 2021] + + tb = tb_amr.merge(tb_total, on=["country", "year", "pathogen"], how="inner") + tb["non_amr_attributable_deaths"] = tb["total_deaths"] - tb["amr_attributable_deaths"] + + tb = geo.harmonize_countries(df=tb, countries_file=paths.country_mapping_path) + + # Reformatting the data so it can be used in stacked bar charts + tb = tb.drop(columns=["country"]).rename(columns={"pathogen": "country"}) + + tb = tb.format(["country", "year"]) + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata + ) + + # Save changes in the new garden dataset. + ds_garden.save() diff --git a/etl/steps/data/garden/antibiotics/2024-12-02/total_pathogen_bloodstream.countries.json b/etl/steps/data/garden/antibiotics/2024-12-02/total_pathogen_bloodstream.countries.json new file mode 100644 index 00000000000..a7f8eced782 --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-02/total_pathogen_bloodstream.countries.json @@ -0,0 +1,3 @@ +{ + "Global": "World" +} \ No newline at end of file diff --git a/etl/steps/data/garden/antibiotics/2024-12-02/total_pathogen_bloodstream.meta.yml b/etl/steps/data/garden/antibiotics/2024-12-02/total_pathogen_bloodstream.meta.yml new file mode 100644 index 00000000000..04044bb693f --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-02/total_pathogen_bloodstream.meta.yml @@ -0,0 +1,46 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Antibiotics + pathogen_type: <% if pathogen_type == "Fungi" %>fungus<% elif pathogen_type == "Viruses" %>virus<% else %><< pathogen_type.lower() >><% endif %> + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + update_period_days: 365 + +tables: + total_pathogen_bloodstream: + variables: + value: + title: Total deaths from << pathogen >> bloodstream infections + unit: deaths + description_short: Estimated number of deaths << pathogen >> bloodstream infections. << pathogen >> is a {definitions.pathogen_type}. + presentation: + title_public: Total deaths from << pathogen >> bloodstream infections + display: + roundingMode: significantFigures + numSignificantFigures: 3 + name: << pathogen >> + upper: + title: Upper bound of total deaths from << pathogen >> bloodstream infections + unit: deaths + description_short: Estimated number of deaths << pathogen >> bloodstream infections. << pathogen >> is a {definitions.pathogen_type}. + presentation: + title_public: Upper bound of total deaths from << pathogen >> bloodstream infections + display: + roundingMode: significantFigures + numSignificantFigures: 3 + name: << pathogen >> + lower: + title: Lower bound of total deaths from << pathogen >> bloodstream infections + unit: deaths + description_short: Estimated number of deaths << pathogen >> bloodstream infections. << pathogen >> is a {definitions.pathogen_type}. + presentation: + title_public: Lower bound of total deaths from << pathogen >> bloodstream infections + display: + roundingMode: significantFigures + numSignificantFigures: 3 + name: << pathogen >> diff --git a/etl/steps/data/garden/antibiotics/2024-11-20/bloodstream_amr.py b/etl/steps/data/garden/antibiotics/2024-12-02/total_pathogen_bloodstream.py similarity index 60% rename from etl/steps/data/garden/antibiotics/2024-11-20/bloodstream_amr.py rename to etl/steps/data/garden/antibiotics/2024-12-02/total_pathogen_bloodstream.py index 4c676495983..ec0b1de3f35 100644 --- a/etl/steps/data/garden/antibiotics/2024-11-20/bloodstream_amr.py +++ b/etl/steps/data/garden/antibiotics/2024-12-02/total_pathogen_bloodstream.py @@ -12,26 +12,23 @@ def run(dest_dir: str) -> None: # Load inputs. # # Load meadow dataset. - ds_meadow = paths.load_dataset("bloodstream_amr") + ds_meadow = paths.load_dataset("total_pathogen_bloodstream") # Read table from meadow dataset. - tb = ds_meadow.read("bloodstream_amr") + tb = ds_meadow.read("total_pathogen_bloodstream") # # Process data. # tb = geo.harmonize_countries(df=tb, countries_file=paths.country_mapping_path) - tb = tb.drop(columns=["age", "sex", "measure", "metric", "infectious_syndrome"]) - tb_amr = tb.drop(columns=["country", "pathogen_type"]).rename(columns={"pathogen": "country"}) - tb_amr = tb_amr.format(["country", "year", "counterfactual"], short_name="amr_entity") - tb = tb.format(["country", "year", "pathogen", "pathogen_type", "counterfactual"]) + tb = tb.format(["country", "year", "pathogen", "pathogen_type"]) # # Save outputs. # # Create a new garden dataset with the same metadata as the meadow dataset. ds_garden = create_dataset( - dest_dir, tables=[tb, tb_amr], check_variables_metadata=True, default_metadata=ds_meadow.metadata + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata ) # Save changes in the new garden dataset. diff --git a/etl/steps/data/garden/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.countries.json b/etl/steps/data/garden/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.countries.json new file mode 100644 index 00000000000..a7f8eced782 --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.countries.json @@ -0,0 +1,3 @@ +{ + "Global": "World" +} \ No newline at end of file diff --git a/etl/steps/data/garden/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.meta.yml b/etl/steps/data/garden/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.meta.yml new file mode 100644 index 00000000000..189b974b11a --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.meta.yml @@ -0,0 +1,42 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Antibiotics + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + update_period_days: 365 + +tables: + total_pathogen_bloodstream_amr: + variables: + amr_attributable_deaths: + title: Total deaths from bloodstream infections attributed to AMR, by pathogen + unit: deaths + description_short: Estimated number of deaths from bloodstream infections that are attributed to antimicrobial resistance. + presentation: + title_public: Total deaths from bloodstream infections attributed to AMR, by pathogen + display: + roundingMode: significantFigures + numSignificantFigures: 3 + non_amr_attributable_deaths: + title: Total global deaths from bloodstream infections not attributed to AMR, by pathogen + unit: deaths + description_short: Estimated number of deaths from bloodstream infections that are not attributed to antimicrobial resistance. + presentation: + title_public: Total global deaths from bloodstream infections not attributed to AMR, by pathogen + display: + roundingMode: significantFigures + numSignificantFigures: 3 + total_deaths: + title: Total global deaths from bloodstream infections + unit: deaths + description_short: Estimated number of deaths from bloodstream infections. + presentation: + title_public: Total global deaths from bloodstream infections + display: + roundingMode: significantFigures + numSignificantFigures: 3 diff --git a/etl/steps/data/garden/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.py b/etl/steps/data/garden/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.py new file mode 100644 index 00000000000..6c09aa1c119 --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.py @@ -0,0 +1,47 @@ +"""Load a meadow dataset and create a garden dataset.""" + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("total_pathogen_bloodstream_amr") + ds_total = paths.load_dataset("total_pathogen_bloodstream") + # Read table from meadow dataset. + tb = ( + ds_meadow.read("total_pathogen_bloodstream_amr") + .drop(columns=["upper", "lower"]) + .rename(columns={"value": "amr_attributable_deaths"}) + ) + tb = geo.harmonize_countries(df=tb, countries_file=paths.country_mapping_path) + + tb_total = ( + ds_total.read("total_pathogen_bloodstream") + .drop(columns=["upper", "lower"]) + .rename(columns={"value": "total_deaths"}) + ) + + tb = tb.merge(tb_total, on=["country", "year", "pathogen", "pathogen_type"], how="inner") + tb["non_amr_attributable_deaths"] = tb["total_deaths"] - tb["amr_attributable_deaths"] + # Process data. + tb = tb.drop(columns=["country", "pathogen_type"]).rename(columns={"pathogen": "country"}) + + tb = tb.format(["country", "year"]) + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata + ) + + # Save changes in the new garden dataset. + ds_garden.save() diff --git a/etl/steps/data/garden/antibiotics/2024-12-03/glass_enrolment.countries.json b/etl/steps/data/garden/antibiotics/2024-12-03/glass_enrolment.countries.json new file mode 100644 index 00000000000..ad9139b95a6 --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-03/glass_enrolment.countries.json @@ -0,0 +1,199 @@ +{ + "Afghanistan": "Afghanistan", + "Albania": "Albania", + "Algeria": "Algeria", + "Andorra": "Andorra", + "Angola": "Angola", + "Antigua and Barbuda": "Antigua and Barbuda", + "Argentina": "Argentina", + "Armenia": "Armenia", + "Australia": "Australia", + "Austria": "Austria", + "Azerbaijan": "Azerbaijan", + "Bahamas": "Bahamas", + "Bahrain": "Bahrain", + "Bangladesh": "Bangladesh", + "Barbados": "Barbados", + "Belarus": "Belarus", + "Belgium": "Belgium", + "Belize": "Belize", + "Benin": "Benin", + "Bhutan": "Bhutan", + "Bolivia (Plurinational State of)": "Bolivia", + "Bosnia and Herzegovina": "Bosnia and Herzegovina", + "Botswana": "Botswana", + "Brazil": "Brazil", + "Brunei Darussalam": "Brunei", + "Bulgaria": "Bulgaria", + "Burkina Faso": "Burkina Faso", + "Burundi": "Burundi", + "Cabo Verde": "Cape Verde", + "Cambodia": "Cambodia", + "Cameroon": "Cameroon", + "Canada": "Canada", + "Central African Republic": "Central African Republic", + "Chad": "Chad", + "Chile": "Chile", + "China": "China", + "Colombia": "Colombia", + "Cook Islands": "Cook Islands", + "Costa Rica": "Costa Rica", + "Croatia": "Croatia", + "Cuba": "Cuba", + "Cyprus": "Cyprus", + "Czech Republic": "Czechia", + "C\u00f4te d\u2019Ivoire": "Cote d'Ivoire", + "Democratic Republic of the Congo": "Democratic Republic of Congo", + "Denmark": "Denmark", + "Djibouti": "Djibouti", + "Dominica": "Dominica", + "Dominican Republic": "Dominican Republic", + "Ecuador": "Ecuador", + "Egypt": "Egypt", + "El Salvador": "El Salvador", + "Equatorial Guinea": "Equatorial Guinea", + "Eritrea": "Eritrea", + "Estonia": "Estonia", + "Eswatini": "Eswatini", + "Ethiopia": "Ethiopia", + "Fiji": "Fiji", + "Finland": "Finland", + "France": "France", + "Gabon": "Gabon", + "Gambia": "Gambia", + "Georgia": "Georgia", + "Germany": "Germany", + "Ghana": "Ghana", + "Greece": "Greece", + "Grenada": "Grenada", + "Guatemala": "Guatemala", + "Guinea": "Guinea", + "Guinea-Bissau": "Guinea-Bissau", + "Guyana": "Guyana", + "Haiti": "Haiti", + "Honduras": "Honduras", + "Hungary": "Hungary", + "Iceland": "Iceland", + "India": "India", + "Indonesia": "Indonesia", + "Iraq": "Iraq", + "Ireland": "Ireland", + "Israel": "Israel", + "Italy": "Italy", + "Jamaica": "Jamaica", + "Japan": "Japan", + "Jordan": "Jordan", + "Kazakhstan": "Kazakhstan", + "Kenya": "Kenya", + "Kiribati": "Kiribati", + "Kosovo": "Kosovo", + "Kuwait": "Kuwait", + "Kyrgyzstan": "Kyrgyzstan", + "Latvia": "Latvia", + "Lebanon": "Lebanon", + "Lesotho": "Lesotho", + "Liberia": "Liberia", + "Libya": "Libya", + "Lithuania": "Lithuania", + "Luxembourg": "Luxembourg", + "Madagascar": "Madagascar", + "Malawi": "Malawi", + "Malaysia": "Malaysia", + "Maldives": "Maldives", + "Mali": "Mali", + "Malta": "Malta", + "Marshall Islands": "Marshall Islands", + "Mauritania": "Mauritania", + "Mauritius": "Mauritius", + "Mexico": "Mexico", + "Micronesia (Federated States of)": "Micronesia (country)", + "Monaco": "Monaco", + "Mongolia": "Mongolia", + "Montenegro": "Montenegro", + "Morocco": "Morocco", + "Mozambique": "Mozambique", + "Myanmar": "Myanmar", + "Namibia": "Namibia", + "Nauru": "Nauru", + "Nepal": "Nepal", + "New Zealand": "New Zealand", + "Nicaragua": "Nicaragua", + "Nigeria": "Nigeria", + "Niue": "Niue", + "North Macedonia": "North Macedonia", + "Norway": "Norway", + "Oman": "Oman", + "Pakistan": "Pakistan", + "Palau": "Palau", + "Palestine": "Palestine", + "Panama": "Panama", + "Papua New Guinea": "Papua New Guinea", + "Paraguay": "Paraguay", + "Peru": "Peru", + "Poland": "Poland", + "Portugal": "Portugal", + "Qatar": "Qatar", + "Republic of Korea": "South Korea", + "Republic of Moldova": "Moldova", + "Romania": "Romania", + "Russian Federation": "Russia", + "Rwanda": "Rwanda", + "Saint Kitts and Nevis": "Saint Kitts and Nevis", + "Saint Lucia": "Saint Lucia", + "Saint Vincent and the Grenadines": "Saint Vincent and the Grenadines", + "Samoa": "Samoa", + "San Marino": "San Marino", + "Sao Tome and Principe": "Sao Tome and Principe", + "Saudi Arabia": "Saudi Arabia", + "Senegal": "Senegal", + "Seychelles": "Seychelles", + "Sierra Leone": "Sierra Leone", + "Singapore": "Singapore", + "Slovakia": "Slovakia", + "Slovenia": "Slovenia", + "Solomon Islands": "Solomon Islands", + "Somalia": "Somalia", + "South Africa": "South Africa", + "South Sudan": "South Sudan", + "Spain": "Spain", + "Sri Lanka": "Sri Lanka", + "Suriname": "Suriname", + "Sweden": "Sweden", + "Switzerland": "Switzerland", + "Syrian Arab Republic": "Syria", + "Tajikistan": "Tajikistan", + "Thailand": "Thailand", + "Timor-Leste": "East Timor", + "Togo": "Togo", + "Tonga": "Tonga", + "Trinidad and Tobago": "Trinidad and Tobago", + "Tunisia": "Tunisia", + "Turkmenistan": "Turkmenistan", + "Tuvalu": "Tuvalu", + "Uganda": "Uganda", + "Ukraine": "Ukraine", + "United Arab Emirates": "United Arab Emirates", + "United Kingdom of Great Britain and Northern Ireland": "United Kingdom", + "United Republic of Tanzania": "Tanzania", + "United States of America": "United States", + "Uruguay": "Uruguay", + "Uzbekistan": "Uzbekistan", + "Vanuatu": "Vanuatu", + "Venezuela (Bolivarian Republic of)": "Venezuela", + "Viet Nam": "Vietnam", + "Yemen": "Yemen", + "Zambia": "Zambia", + "Zimbabwe": "Zimbabwe", + "Comoros ": "Comoros", + "Congo ": "Congo", + "Democratic People\u2019s Republic of Korea": "North Korea", + "Hong Kong SAR (China)": "Hong Kong", + "Iran (Islamic Republic)": "Iran", + "Lao People\u2019s Democratic Republic": "Laos", + "Netherlands ": "Netherlands", + "Niger ": "Niger", + "Philippines ": "Philippines", + "Serbia ": "Serbia", + "Sudan ": "Sudan", + "T\u00fcrkiye": "Turkey" +} \ No newline at end of file diff --git a/etl/steps/data/garden/antibiotics/2024-12-03/glass_enrolment.meta.yml b/etl/steps/data/garden/antibiotics/2024-12-03/glass_enrolment.meta.yml new file mode 100644 index 00000000000..d830f8aaebc --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-03/glass_enrolment.meta.yml @@ -0,0 +1,20 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Antibiotics + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + update_period_days: 365 + +tables: + glass_enrolment: + variables: + enrolment: + title: Which countries have enrolled in the Global Antimicrobial Resistance Surveillance System (GLASS)? + description_short: "Global Antimicrobial Resistance Surveillance System participation status of each country." + unit: "" + short_unit: "" diff --git a/etl/steps/data/garden/antibiotics/2024-12-03/glass_enrolment.py b/etl/steps/data/garden/antibiotics/2024-12-03/glass_enrolment.py new file mode 100644 index 00000000000..06c4fc342e6 --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-03/glass_enrolment.py @@ -0,0 +1,72 @@ +"""Load a meadow dataset and create a garden dataset.""" + +import numpy as np +import pandas as pd +from owid.catalog import Table + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("glass_enrolment") + + # Read table from meadow dataset. + tb = ds_meadow.read("glass_enrolment") + + # + # Process data. + # + tb = geo.harmonize_countries(tb, countries_file=paths.country_mapping_path) + origins = tb["amc"].metadata.origins + + # Make data meaningful. + tb = tb[["country", "year", "amr", "amc"]] + # Check there's no weird values, it should be only Y and NA + assert len(tb["amr"].unique()) == 2, "amr column should have only two unique values" + assert len(tb["amc"].unique()) == 2, "amc column should have only two unique values" + tb = combine_data(tb) + tb = tb.drop(columns=["amr", "amc"]) + tb["enrolment"].metadata.origins = origins + + tb = tb.format(["country", "year"]) + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata + ) + + # Save changes in the new garden dataset. + ds_garden.save() + + +def combine_data(tb: Table) -> Table: + """Combine data the amr and amc columns into a single column.""" + # Define conditions + tb["amr"] = tb["amr"].fillna("") + tb["amc"] = tb["amc"].fillna("") + conditions = [ + (tb["amr"] == "Y") & (tb["amc"] == "Y"), # Both AMR and AMC + (tb["amr"] == "Y") & (tb["amc"] != "Y"), # AMR only + (tb["amr"] != "Y") & (tb["amc"] == "Y"), # AMC only + (tb["amr"] != "Y") & (tb["amc"] != "Y"), # Neither + ] + + # Define corresponding outputs + choices = ["Both", "AMR only", "AMC only", "Neither"] + + # Apply row-wise conditions + tb["enrolment"] = np.select(conditions, choices, default=pd.NA) + assert all(tb["enrolment"].notna()), "There should be no missing values in the enrolment column" + + return tb diff --git a/etl/steps/data/garden/antibiotics/2024-12-04/microbe_total_pathogens.countries.json b/etl/steps/data/garden/antibiotics/2024-12-04/microbe_total_pathogens.countries.json new file mode 100644 index 00000000000..a7f8eced782 --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-04/microbe_total_pathogens.countries.json @@ -0,0 +1,3 @@ +{ + "Global": "World" +} \ No newline at end of file diff --git a/etl/steps/data/garden/antibiotics/2024-12-04/microbe_total_pathogens.meta.yml b/etl/steps/data/garden/antibiotics/2024-12-04/microbe_total_pathogens.meta.yml new file mode 100644 index 00000000000..e29b06ed86b --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-04/microbe_total_pathogens.meta.yml @@ -0,0 +1,46 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Antibiotics + pathogen_type: <% if pathogen_type == "Fungi" %>fungus<% elif pathogen_type == "Viruses" %>virus<% else %><< pathogen_type.lower() >><% endif %> + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + update_period_days: 365 + +tables: + microbe_total_pathogens: + variables: + value: + title: Total deaths from << pathogen >> infections + unit: deaths + description_short: Estimated number of deaths << pathogen >> infections. << pathogen >> is a {definitions.pathogen_type}. + presentation: + title_public: Total deaths from << pathogen >> infections + display: + roundingMode: significantFigures + numSignificantFigures: 3 + name: << pathogen >> + upper: + title: Upper bound of total deaths from << pathogen >> infections + unit: deaths + description_short: Estimated number of deaths << pathogen >> infections. << pathogen >> is a {definitions.pathogen_type}. + presentation: + title_public: Upper bound of total deaths from << pathogen >> infections + display: + roundingMode: significantFigures + numSignificantFigures: 3 + name: << pathogen >> + lower: + title: Lower bound of total deaths from << pathogen >> infections + unit: deaths + description_short: Estimated number of deaths << pathogen >> infections. << pathogen >> is a {definitions.pathogen_type}. + presentation: + title_public: Lower bound of total deaths from << pathogen >> infections + display: + roundingMode: significantFigures + numSignificantFigures: 3 + name: << pathogen >> diff --git a/etl/steps/data/garden/antibiotics/2024-12-04/microbe_total_pathogens.py b/etl/steps/data/garden/antibiotics/2024-12-04/microbe_total_pathogens.py new file mode 100644 index 00000000000..b6883c23419 --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-04/microbe_total_pathogens.py @@ -0,0 +1,35 @@ +"""Load a meadow dataset and create a garden dataset.""" + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("microbe_total_pathogens") + + # Read table from meadow dataset. + tb = ds_meadow.read("microbe_total_pathogens") + + # + # Process data. + # + tb = geo.harmonize_countries(df=tb, countries_file=paths.country_mapping_path) + tb = tb.format(["country", "year", "pathogen", "pathogen_type"]) + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata + ) + + # Save changes in the new garden dataset. + ds_garden.save() diff --git a/etl/steps/data/garden/antibiotics/2024-12-04/microbe_total_pathogens_amr.countries.json b/etl/steps/data/garden/antibiotics/2024-12-04/microbe_total_pathogens_amr.countries.json new file mode 100644 index 00000000000..a7f8eced782 --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-04/microbe_total_pathogens_amr.countries.json @@ -0,0 +1,3 @@ +{ + "Global": "World" +} \ No newline at end of file diff --git a/etl/steps/data/garden/antibiotics/2024-12-04/microbe_total_pathogens_amr.meta.yml b/etl/steps/data/garden/antibiotics/2024-12-04/microbe_total_pathogens_amr.meta.yml new file mode 100644 index 00000000000..2edb137309c --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-04/microbe_total_pathogens_amr.meta.yml @@ -0,0 +1,42 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Antibiotics + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + update_period_days: 365 + +tables: + microbe_total_pathogens_amr: + variables: + amr_attributable_deaths: + title: Total deaths from infections attributed to AMR, by pathogen + unit: deaths + description_short: Estimated number of deaths from infections that are attributed to antimicrobial resistance. + presentation: + title_public: Total deaths from infections attributed to AMR, by pathogen + display: + roundingMode: significantFigures + numSignificantFigures: 3 + non_amr_attributable_deaths: + title: Total global deaths from infections not attributed to AMR, by pathogen + unit: deaths + description_short: Estimated number of deaths from infections that are not attributed to antimicrobial resistance. + presentation: + title_public: Total global deaths from infections not attributed to AMR, by pathogen + display: + roundingMode: significantFigures + numSignificantFigures: 3 + total_deaths: + title: Total global deaths from infections + unit: deaths + description_short: Estimated number of deaths from infections. + presentation: + title_public: Total global deaths from infections + display: + roundingMode: significantFigures + numSignificantFigures: 3 diff --git a/etl/steps/data/garden/antibiotics/2024-12-04/microbe_total_pathogens_amr.py b/etl/steps/data/garden/antibiotics/2024-12-04/microbe_total_pathogens_amr.py new file mode 100644 index 00000000000..0c71e6b28f7 --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-04/microbe_total_pathogens_amr.py @@ -0,0 +1,53 @@ +"""Load a meadow dataset and create a garden dataset.""" + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("microbe_total_pathogens_amr") + ds_total = paths.load_dataset("microbe_total_pathogens") + + # Read table from meadow dataset. + tb = ( + ds_meadow.read("microbe_total_pathogens_amr") + .drop(columns=["upper", "lower"]) + .rename(columns={"value": "amr_attributable_deaths"}) + ) + tb_total = ( + ds_total.read("microbe_total_pathogens") + .drop(columns=["upper", "lower"]) + .rename(columns={"value": "total_deaths"}) + ) + # + # Process data. + # + tb = geo.harmonize_countries( + df=tb, + countries_file=paths.country_mapping_path, + ) + + tb = tb.merge(tb_total, on=["country", "year", "pathogen", "pathogen_type"], how="inner") + tb["non_amr_attributable_deaths"] = tb["total_deaths"] - tb["amr_attributable_deaths"] + # Process data. + tb = tb.drop(columns=["country", "pathogen_type"]).rename(columns={"pathogen": "country"}) + + tb = tb.format(["country", "year"]) + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata + ) + + # Save changes in the new garden dataset. + ds_garden.save() diff --git a/etl/steps/data/garden/antibiotics/2024-12-05/microbe_neonatal_total_amr.countries.json b/etl/steps/data/garden/antibiotics/2024-12-05/microbe_neonatal_total_amr.countries.json new file mode 100644 index 00000000000..a7f8eced782 --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-05/microbe_neonatal_total_amr.countries.json @@ -0,0 +1,3 @@ +{ + "Global": "World" +} \ No newline at end of file diff --git a/etl/steps/data/garden/antibiotics/2024-11-24/total_syndrome.meta.yml b/etl/steps/data/garden/antibiotics/2024-12-05/microbe_neonatal_total_amr.meta.yml similarity index 56% rename from etl/steps/data/garden/antibiotics/2024-11-24/total_syndrome.meta.yml rename to etl/steps/data/garden/antibiotics/2024-12-05/microbe_neonatal_total_amr.meta.yml index 812eeaf01dc..c9de44d96cd 100644 --- a/etl/steps/data/garden/antibiotics/2024-11-24/total_syndrome.meta.yml +++ b/etl/steps/data/garden/antibiotics/2024-12-05/microbe_neonatal_total_amr.meta.yml @@ -5,38 +5,38 @@ definitions: topic_tags: - Antibiotics - # Learn more about the available fields: # http://docs.owid.io/projects/etl/architecture/metadata/reference/ dataset: update_period_days: 365 - tables: - total_syndrome: + microbe_neonatal_total_amr: variables: - value: - title: Global neonatal deaths from << infectious_syndrome.lower() >> infections + amr_attributable_deaths: + title: Neonatal deaths from infections attributed to AMR, by pathogen unit: deaths - description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from << infectious_syndrome.lower() >>. + description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from infections that are attributed to antimicrobial resistance. + presentation: + title_public: Neonatal deaths from infections attributed to AMR, by pathogen display: roundingMode: significantFigures numSignificantFigures: 3 - name: << infectious_syndrome >> - upper: - title: Upper bound of global neonatal deaths from << infectious_syndrome.lower() >> infections + non_amr_attributable_deaths: + title: Neonatal global deaths from infections not attributed to AMR, by pathogen unit: deaths - description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from << infectious_syndrome.lower() >>. + description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from infections that are not attributed to antimicrobial resistance. + presentation: + title_public: Neonatal global deaths from infections not attributed to AMR, by pathogen display: roundingMode: significantFigures numSignificantFigures: 3 - name: << infectious_syndrome >> - lower: - title: Lower bound of global neonatal deaths from << infectious_syndrome.lower() >> infections + total_deaths: + title: Neonatal global deaths from infections unit: deaths - description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from << infectious_syndrome.lower() >>. + description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from infections. + presentation: + title_public: Neonatal global deaths from infections display: - numSignificantFigures: 3 roundingMode: significantFigures - name: << infectious_syndrome >> - + numSignificantFigures: 3 diff --git a/etl/steps/data/garden/antibiotics/2024-12-05/microbe_neonatal_total_amr.py b/etl/steps/data/garden/antibiotics/2024-12-05/microbe_neonatal_total_amr.py new file mode 100644 index 00000000000..0bbaa192ae6 --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-05/microbe_neonatal_total_amr.py @@ -0,0 +1,45 @@ +"""Load a meadow dataset and create a garden dataset.""" + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("microbe_neonatal_total_amr") + ds_total = paths.load_dataset("microbe") + + # Read table from meadow dataset. + tb = ( + ds_meadow.read("microbe_neonatal_total_amr") + .drop(columns=["upper", "lower"]) + .rename(columns={"value": "amr_attributable_deaths"}) + ) + tb_total = ds_total.read("microbe").drop(columns=["upper", "lower"]).rename(columns={"value": "total_deaths"}) + + # + # Process data. + # + tb = geo.harmonize_countries(df=tb, countries_file=paths.country_mapping_path) + + tb = tb.merge(tb_total, on=["country", "year", "pathogen", "pathogen_type"], how="inner") + tb["non_amr_attributable_deaths"] = tb["total_deaths"] - tb["amr_attributable_deaths"] + tb = tb.drop(columns=["country", "pathogen_type"]).rename(columns={"pathogen": "country"}) + tb = tb.format(["country", "year"]) + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata + ) + + # Save changes in the new garden dataset. + ds_garden.save() diff --git a/etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.countries.json b/etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.countries.json new file mode 100644 index 00000000000..a7f8eced782 --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.countries.json @@ -0,0 +1,3 @@ +{ + "Global": "World" +} \ No newline at end of file diff --git a/etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.meta.yml b/etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.meta.yml new file mode 100644 index 00000000000..37ceb3b7b84 --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.meta.yml @@ -0,0 +1,45 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Antibiotics + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + update_period_days: 365 + +tables: + microbe_total_deaths_by_syndrome: + variables: + value: + title: Total deaths from << infectious_syndrome.lower() >> + unit: deaths + description_short: Estimated number of deaths from << infectious_syndrome.lower() >>. + presentation: + title_public: Total deaths from << infectious_syndrome.lower() >> + display: + roundingMode: significantFigures + numSignificantFigures: 3 + name: << pathogen >> + lower: + title: Lower bound of deaths from << infectious_syndrome.lower() >> + unit: deaths + description_short: Estimated number of deaths from << infectious_syndrome.lower() >>. + presentation: + title_public: Lower bound of deaths from << infectious_syndrome.lower() >> + display: + roundingMode: significantFigures + numSignificantFigures: 3 + name: << pathogen >> + upper: + title: Upper bound of deaths from << infectious_syndrome.lower() >> + unit: deaths + description_short: Estimated number of deaths from << infectious_syndrome.lower() >>. + presentation: + title_public: Upper bound of deaths from << infectious_syndrome.lower() >> + display: + roundingMode: significantFigures + numSignificantFigures: 3 + name: << pathogen >> diff --git a/etl/steps/data/garden/antibiotics/2024-11-24/total_syndrome.py b/etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.py similarity index 86% rename from etl/steps/data/garden/antibiotics/2024-11-24/total_syndrome.py rename to etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.py index fb418bd8636..1659dd7480e 100644 --- a/etl/steps/data/garden/antibiotics/2024-11-24/total_syndrome.py +++ b/etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.py @@ -12,10 +12,10 @@ def run(dest_dir: str) -> None: # Load inputs. # # Load meadow dataset. - ds_meadow = paths.load_dataset("total_syndrome") + ds_meadow = paths.load_dataset("microbe_total_deaths_by_syndrome") # Read table from meadow dataset. - tb = ds_meadow.read("total_syndrome") + tb = ds_meadow.read("microbe_total_deaths_by_syndrome") # # Process data. diff --git a/etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.countries.json b/etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.countries.json new file mode 100644 index 00000000000..40711a8e695 --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.countries.json @@ -0,0 +1,3 @@ +{ + "Global": "World" +} diff --git a/etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.meta.yml b/etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.meta.yml new file mode 100644 index 00000000000..55fdd39bf53 --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.meta.yml @@ -0,0 +1,42 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Antibiotics + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + update_period_days: 365 + +tables: + microbe_total_deaths_by_syndrome_amr: + variables: + amr_attributable_deaths: + title: Total deaths from infections attributed to AMR, by syndrome + unit: deaths + description_short: Estimated number of deaths each year from infections that are attributed to antimicrobial resistance. + presentation: + title_public: Total deaths from infections attributed to AMR, by syndrome + display: + roundingMode: significantFigures + numSignificantFigures: 3 + non_amr_attributable_deaths: + title: Total deaths from infections not attributed to AMR, by syndrome + unit: deaths + description_short: Estimated number of deaths each year from infections that are not attributed to antimicrobial resistance. + presentation: + title_public: Total deaths from infections not attributed to AMR, by syndrome + display: + roundingMode: significantFigures + numSignificantFigures: 3 + total_deaths: + title: Total deaths from infections, by syndrome + unit: deaths + description_short: Estimated number deaths each year from infections. + presentation: + title_public: Total deaths from infections, by syndrome + display: + roundingMode: significantFigures + numSignificantFigures: 3 diff --git a/etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.py b/etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.py new file mode 100644 index 00000000000..ef3048c297c --- /dev/null +++ b/etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.py @@ -0,0 +1,93 @@ +"""Load a meadow dataset and create a garden dataset.""" + +from owid.catalog import Table + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("microbe_total_deaths_by_syndrome_amr") + ds_total = paths.load_dataset("microbe_total_deaths_by_syndrome") + + # Read table from meadow dataset. + tb = ( + ds_meadow.read("microbe_total_deaths_by_syndrome_amr") + .drop(columns=["upper", "lower"]) + .rename(columns={"value": "amr_attributable_deaths"}) + ) + + tb_total = ( + ds_total.read("microbe_total_deaths_by_syndrome") + .drop(columns=["upper", "lower"]) + .rename(columns={"value": "total_deaths"}) + ) + # + # Process data. + # + tb = geo.harmonize_countries(df=tb, countries_file=paths.country_mapping_path) + + tb = tb.merge(tb_total, on=["country", "year", "infectious_syndrome"], how="inner") + + tb = rename_syndromes(tb) + tb["non_amr_attributable_deaths"] = tb["total_deaths"] - tb["amr_attributable_deaths"] + tb = tb.drop(columns=["country"]).rename(columns={"infectious_syndrome": "country"}) + tb = tb.format(["country", "year"]) + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata + ) + + # Save changes in the new garden dataset. + ds_garden.save() + + +def rename_syndromes(tb: Table) -> Table: + """ + Rename syndromes to be shorter for use in stacked bar charts. + Ensure all infectious syndromes are replaced. + """ + name_dict = { + "Bloodstream infections": "Bloodstream infections", + "Lower respiratory infections": "Lower respiratory infections", + "Diarrhea": "Diarrhea", + "Meningitis": "Meningitis", + "Infections of the skin and subcutaneous systems": "Skin infections", + "Urinary tract infections and pyelonephritis": "Kidney and urinary tract infections", + "Peritoneal and intra-abdominal infections": "Abdominal infections", + "Tuberculosis": "Tuberculosis", + "Endocarditis": "Endocarditis", + "Typhoid fever, paratyphoid fever, and invasive non-typhoidal Salmonella": "Typhoid, paratyphoid, and iNTS", + "Infections of bones, joints, and related organs": "Bone and joint infections", + "Other unspecified site infections": "Other infections", + "Other parasitic infections": "Other parasitic infections", + "Oral infections": "Oral infections", + "Myelitis, meningoencephalitis, and other central nervous system infections": "Central nervous system infections", + "Upper respiratory infections": "Upper respiratory infections", + "Hepatitis": "Hepatitis", + "Eye infections": "Eye infections", + "Encephalitis": "Encephalitis", + "Carditis, myocarditis, and pericarditis": "Heart inflammation", + "Sexually transmitted infections": "Sexually transmitted infections", + } + + # Find unmatched syndromes + unmatched_syndromes = set(tb["infectious_syndrome"].unique()) - set(name_dict.keys()) + if unmatched_syndromes: + raise ValueError(f"The following syndromes were not found in the name dictionary: {unmatched_syndromes}") + + # Replace syndromes + tb["infectious_syndrome"] = tb["infectious_syndrome"].replace(name_dict) + + return tb diff --git a/etl/steps/data/garden/artificial_intelligence/2024-01-30/yougov_job_automation.py b/etl/steps/data/garden/artificial_intelligence/2024-01-30/yougov_job_automation.py index 7fbf4c57a40..1dfd8c3dbeb 100644 --- a/etl/steps/data/garden/artificial_intelligence/2024-01-30/yougov_job_automation.py +++ b/etl/steps/data/garden/artificial_intelligence/2024-01-30/yougov_job_automation.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - import pandas as pd from structlog import get_logger diff --git a/etl/steps/data/garden/artificial_intelligence/2024-01-30/yougov_robots.py b/etl/steps/data/garden/artificial_intelligence/2024-01-30/yougov_robots.py index 6ff1a81ea25..25ddfa2f523 100644 --- a/etl/steps/data/garden/artificial_intelligence/2024-01-30/yougov_robots.py +++ b/etl/steps/data/garden/artificial_intelligence/2024-01-30/yougov_robots.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - import pandas as pd from structlog import get_logger diff --git a/etl/steps/data/garden/artificial_intelligence/2024-02-15/epoch_llms.py b/etl/steps/data/garden/artificial_intelligence/2024-02-15/epoch_llms.py index ab12e32d1a0..e222dfd5f39 100644 --- a/etl/steps/data/garden/artificial_intelligence/2024-02-15/epoch_llms.py +++ b/etl/steps/data/garden/artificial_intelligence/2024-02-15/epoch_llms.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/garden/artificial_intelligence/2024-04-02/dynabench.py b/etl/steps/data/garden/artificial_intelligence/2024-04-02/dynabench.py index c0d48e24e02..dc2228e5a48 100644 --- a/etl/steps/data/garden/artificial_intelligence/2024-04-02/dynabench.py +++ b/etl/steps/data/garden/artificial_intelligence/2024-04-02/dynabench.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch.py b/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch.py index 1958173d45b..55ae25212bd 100644 --- a/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch.py +++ b/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + import pandas as pd from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch_aggregates_affiliation.py b/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch_aggregates_affiliation.py index 19a4d7130e5..131930a13e6 100644 --- a/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch_aggregates_affiliation.py +++ b/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch_aggregates_affiliation.py @@ -1,4 +1,4 @@ -""" Generate aggregated table for total yearly and cumulative number of notable AI systems in each category of researcher affiliation.""" +"""Generate aggregated table for total yearly and cumulative number of notable AI systems in each category of researcher affiliation.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch_aggregates_countries.py b/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch_aggregates_countries.py index 0ec9011362f..2b67bb32122 100644 --- a/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch_aggregates_countries.py +++ b/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch_aggregates_countries.py @@ -1,4 +1,5 @@ -""" Generate aggregated table for total yearly and cumulative number of notable AI systems in each country.""" +"""Generate aggregated table for total yearly and cumulative number of notable AI systems in each country.""" + import shared as sh from etl.data_helpers import geo diff --git a/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch_aggregates_domain.py b/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch_aggregates_domain.py index a3a8bee064c..be9bbb9f9e8 100644 --- a/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch_aggregates_domain.py +++ b/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch_aggregates_domain.py @@ -1,4 +1,5 @@ -""" Generate aggregated table for total yearly and cumulative number of notable AI systems for each domain.""" +"""Generate aggregated table for total yearly and cumulative number of notable AI systems for each domain.""" + import shared as sh from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch_aggregates_organizations.py b/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch_aggregates_organizations.py index 64166f6a11f..83fe4868c06 100644 --- a/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch_aggregates_organizations.py +++ b/etl/steps/data/garden/artificial_intelligence/2024-06-03/epoch_aggregates_organizations.py @@ -1,4 +1,5 @@ -""" Generate aggregated table for total yearly and cumulative number of notable AI systems for each organization.""" +"""Generate aggregated table for total yearly and cumulative number of notable AI systems for each organization.""" + import shared as sh from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/garden/artificial_intelligence/2024-06-19/epoch_compute_intensive.py b/etl/steps/data/garden/artificial_intelligence/2024-06-19/epoch_compute_intensive.py index b13c7293c38..645ffdbce50 100644 --- a/etl/steps/data/garden/artificial_intelligence/2024-06-19/epoch_compute_intensive.py +++ b/etl/steps/data/garden/artificial_intelligence/2024-06-19/epoch_compute_intensive.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + import pandas as pd from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/garden/artificial_intelligence/2024-06-19/epoch_compute_intensive_countries.py b/etl/steps/data/garden/artificial_intelligence/2024-06-19/epoch_compute_intensive_countries.py index f0fdd93956a..89e4430781a 100644 --- a/etl/steps/data/garden/artificial_intelligence/2024-06-19/epoch_compute_intensive_countries.py +++ b/etl/steps/data/garden/artificial_intelligence/2024-06-19/epoch_compute_intensive_countries.py @@ -1,4 +1,5 @@ -""" Generate aggregated table for total yearly and cumulative number of notable AI systems in each country.""" +"""Generate aggregated table for total yearly and cumulative number of notable AI systems in each country.""" + import shared as sh from etl.data_helpers import geo diff --git a/etl/steps/data/garden/artificial_intelligence/2024-06-19/epoch_compute_intensive_domain.py b/etl/steps/data/garden/artificial_intelligence/2024-06-19/epoch_compute_intensive_domain.py index 1cf3ba51a9a..b7af77bce9f 100644 --- a/etl/steps/data/garden/artificial_intelligence/2024-06-19/epoch_compute_intensive_domain.py +++ b/etl/steps/data/garden/artificial_intelligence/2024-06-19/epoch_compute_intensive_domain.py @@ -1,4 +1,5 @@ -""" Generate aggregated table for total yearly and cumulative number of notable AI systems for each domain.""" +"""Generate aggregated table for total yearly and cumulative number of notable AI systems for each domain.""" + import shared as sh from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/garden/artificial_intelligence/2024-06-19/epoch_compute_intensive_organizations.py b/etl/steps/data/garden/artificial_intelligence/2024-06-19/epoch_compute_intensive_organizations.py index 7a440eef7b9..4f49e0a804d 100644 --- a/etl/steps/data/garden/artificial_intelligence/2024-06-19/epoch_compute_intensive_organizations.py +++ b/etl/steps/data/garden/artificial_intelligence/2024-06-19/epoch_compute_intensive_organizations.py @@ -1,4 +1,5 @@ -""" Generate aggregated table for total yearly and cumulative number of notable AI systems for each organization.""" +"""Generate aggregated table for total yearly and cumulative number of notable AI systems for each organization.""" + import shared as sh from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/garden/artificial_intelligence/2024-07-10/epoch.py b/etl/steps/data/garden/artificial_intelligence/2024-07-10/epoch.py index 1958173d45b..55ae25212bd 100644 --- a/etl/steps/data/garden/artificial_intelligence/2024-07-10/epoch.py +++ b/etl/steps/data/garden/artificial_intelligence/2024-07-10/epoch.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + import pandas as pd from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/garden/artificial_intelligence/2024-07-10/epoch_aggregates_affiliation.py b/etl/steps/data/garden/artificial_intelligence/2024-07-10/epoch_aggregates_affiliation.py index 19a4d7130e5..131930a13e6 100644 --- a/etl/steps/data/garden/artificial_intelligence/2024-07-10/epoch_aggregates_affiliation.py +++ b/etl/steps/data/garden/artificial_intelligence/2024-07-10/epoch_aggregates_affiliation.py @@ -1,4 +1,4 @@ -""" Generate aggregated table for total yearly and cumulative number of notable AI systems in each category of researcher affiliation.""" +"""Generate aggregated table for total yearly and cumulative number of notable AI systems in each category of researcher affiliation.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/garden/artificial_intelligence/2024-07-10/epoch_aggregates_countries.py b/etl/steps/data/garden/artificial_intelligence/2024-07-10/epoch_aggregates_countries.py index 0ec9011362f..2b67bb32122 100644 --- a/etl/steps/data/garden/artificial_intelligence/2024-07-10/epoch_aggregates_countries.py +++ b/etl/steps/data/garden/artificial_intelligence/2024-07-10/epoch_aggregates_countries.py @@ -1,4 +1,5 @@ -""" Generate aggregated table for total yearly and cumulative number of notable AI systems in each country.""" +"""Generate aggregated table for total yearly and cumulative number of notable AI systems in each country.""" + import shared as sh from etl.data_helpers import geo diff --git a/etl/steps/data/garden/artificial_intelligence/2024-07-10/epoch_aggregates_domain.py b/etl/steps/data/garden/artificial_intelligence/2024-07-10/epoch_aggregates_domain.py index a3a8bee064c..be9bbb9f9e8 100644 --- a/etl/steps/data/garden/artificial_intelligence/2024-07-10/epoch_aggregates_domain.py +++ b/etl/steps/data/garden/artificial_intelligence/2024-07-10/epoch_aggregates_domain.py @@ -1,4 +1,5 @@ -""" Generate aggregated table for total yearly and cumulative number of notable AI systems for each domain.""" +"""Generate aggregated table for total yearly and cumulative number of notable AI systems for each domain.""" + import shared as sh from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/garden/artificial_intelligence/2024-11-03/epoch_regressions.py b/etl/steps/data/garden/artificial_intelligence/2024-11-03/epoch_regressions.py index 3543526e319..6d30589f245 100644 --- a/etl/steps/data/garden/artificial_intelligence/2024-11-03/epoch_regressions.py +++ b/etl/steps/data/garden/artificial_intelligence/2024-11-03/epoch_regressions.py @@ -10,6 +10,7 @@ # Get paths and naming conventions for current step. paths = PathFinder(__file__) +# Constants for defining the time periods DL_ERA_START = 2010 START_DATE = 1950 END_DATE = 2025.2 @@ -72,67 +73,63 @@ def fit_exponential(models, metric): def run_regression(tb): """Run regression analysis on the given table and return the updated table.""" + # Add fractional year for sorting and processing publication_dates = tb["publication_date"] tb.loc[:, "frac_year"] = ( publication_dates.dt.year + (publication_dates.dt.month - 1) / 12 + (publication_dates.dt.day - 1) / 365 ) tb = tb.sort_values(by="frac_year") + # Define periods dynamically + periods = { + f"{START_DATE}–{DL_ERA_START}": (tb["frac_year"] < DL_ERA_START), + f"{DL_ERA_START}–{int(END_DATE)}": ((tb["frac_year"] >= DL_ERA_START) & (tb["frac_year"] < END_DATE)), + } + # Define year grids dynamically + year_grids = { + f"{START_DATE}–{DL_ERA_START}": np.array([START_DATE, DL_ERA_START]), + f"{DL_ERA_START}–{int(END_DATE)}": np.array([DL_ERA_START, END_DATE]), + } + metrics = ["training_computation_petaflop", "parameters", "training_dataset_size__datapoints"] new_tables = [] - for m, metric in enumerate(metrics): + for metric in metrics: # Filter out models without the metric information tb_metric = tb[pd.notnull(tb[metric])] - - # Fit exponential models for pre-DL and DL eras - pre_dl_models = tb_metric[tb_metric["frac_year"] < DL_ERA_START] - pre_dl_fit = fit_exponential(pre_dl_models, metric) - pre_dl_oom_per_year = pre_dl_fit[1] - - dl_models = tb_metric[tb_metric["frac_year"] >= DL_ERA_START] - dl_fit = fit_exponential(dl_models, metric) - dl_oom_per_year = dl_fit[1] - - # Log the results - pre_dl_info = f"{10**pre_dl_oom_per_year:.1f}x/year" - dl_info = f"{10**dl_oom_per_year:.1f}x/year" - paths.log.info(f"Pre Deep Learning Era ({metric}): {pre_dl_info}") - paths.log.info(f"Deep Learning Era ({metric}): {dl_info}") - - # Define the year grids for the periods 1950 to 2010 and 2010 to 2025 with just two points - pre_dl_year_grid = np.array([START_DATE, DL_ERA_START]) - dl_year_grid = np.array([DL_ERA_START, END_DATE]) - - # Calculate the lines for each period using the fitted exponential models - pre_dl_line = 10 ** (pre_dl_fit[0] + pre_dl_year_grid * pre_dl_fit[1]) - dl_line = 10 ** (dl_fit[0] + dl_year_grid * dl_fit[1]) - - # Create new DataFrames for pre-deep learning and deep learning era trends with only necessary columns - pre_dl_df = pd.DataFrame( - { - "days_since_1949": [ - tb_metric["days_since_1949"].min(), - tb_metric[tb_metric["frac_year"] < DL_ERA_START]["days_since_1949"].max(), - ], - f"{metric}": [pre_dl_line[0], pre_dl_line[-1]], - "system": [f"{pre_dl_info}"] * 2, - } - ) - - dl_df = pd.DataFrame( - { - "days_since_1949": [ - tb_metric[tb_metric["frac_year"] >= DL_ERA_START]["days_since_1949"].min(), - tb_metric["days_since_1949"].max(), - ], - f"{metric}": [dl_line[0], dl_line[-1]], - "system": [f"{dl_info}"] * 2, - } - ) - - # Combine the pre-deep learning and deep learning era DataFrames - df_combined = pd.concat([pre_dl_df, dl_df], ignore_index=True) + dfs = [] + + for period_name, condition in periods.items(): + # Subset data for the current period + period_data = tb_metric[condition] + + # Fit exponential model + fit = fit_exponential(period_data, metric) + oom_per_year = fit[1] + info = f"{10**oom_per_year:.1f}x/year" + + # Log the results + paths.log.info(f"{period_name} ({metric}): {info}") + + # Calculate the regression line for the current period + year_grid = year_grids[period_name] + line = 10 ** (fit[0] + year_grid * fit[1]) + + # Create DataFrame for the current period + df = pd.DataFrame( + { + "days_since_1949": [ + period_data["days_since_1949"].min(), + period_data["days_since_1949"].max(), + ], + f"{metric}": [line[0], line[-1]], + "system": [f"{info} between {period_name}"] * 2, + } + ) + dfs.append(df) + + # Combine the DataFrames for all periods for the current metric + df_combined = pd.concat(dfs, ignore_index=True) new_tables.append(df_combined) # Merge all the new DataFrames diff --git a/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch.meta.yml b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch.meta.yml new file mode 100644 index 00000000000..c4764e0418e --- /dev/null +++ b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch.meta.yml @@ -0,0 +1,98 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + processing_level: major + presentation: + topic_tags: + - Artificial Intelligence +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/dataset/ +dataset: + update_period_days: 31 + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/tables/ +tables: + epoch: + variables: + domain: + title: Domain + unit: '' + short_unit: '' + description_short: Refers to the specific area, application, or field in which an AI model is designed to operate. + description_processing: |- + In cases where multiple domains were associated with a model, we consolidated these entries under the label "Multiple domains". We also identified domains associated with fewer than 20 notable systems and grouped these under the category 'Other'. + display: + zeroDay: '1949-01-01' + yearIsDay: true + + organization_categorization: + title: Researcher affiliation + unit: '' + short_unit: '' + description_short: Describes the sector where the authors of an AI model have their primary affiliations. + description_from_producer: |- + Systems are categorized as “Industry” if their authors are affiliated with private sector organizations, “Academia” if the authors are affiliated with universities or academic institutions, or “Industry - Academia Collaboration” when at least 30% of the authors are from each. + + parameters: + title: Number of parameters + unit: '' + description_short: Total number of learnable variables or weights that the model contains. Parameters are adjusted during the training process to optimize the model's performance. + description_key: + - Parameters are internal variables that machine learning models adjust during their training process to improve their ability to make accurate predictions. They act as the model's "knobs" that are fine-tuned based on the provided data. In deep learning, a subset of artificial intelligence (AI), parameters primarily consist of the weights assigned to the connections between the small processing units called neurons. Picture a vast network of interconnected neurons where the strength of each connection represents a parameter. + + - The total number of parameters in a model is influenced by various factors. The model's structure and the number of “layers” of neurons play a significant role. Generally, more complex models with additional layers tend to have a higher number of parameters. Special components of specific deep learning architectures can further contribute to the overall parameter count. + + - Understanding the number of parameters in a model is crucial to design effective models. More parameters can help the model understand complex data patterns, potentially leading to higher accuracy. However, there's a fine balance to strike. If a model has too many parameters, it risks memorizing the specific examples in its training data rather than learning their underlying patterns. Consequently, it may perform poorly when presented with new, unseen data. Achieving the right balance of parameters is a critical consideration in model development. + + - In recent times, the AI community has witnessed the emergence of what are often referred to as "giant models." These models boast an astounding number of parameters, reaching into the billions or even trillions. While these huge models have achieved remarkable performance, they have a significant computational cost. Effectively managing and training such large-scale models has become a prominent and active area of research and discussion within the AI field. + + display: + numDecimalPlaces: 0 + zeroDay: '1949-01-01' + yearIsDay: true + + training_dataset_size__datapoints: + title: Training dataset size + unit: 'datapoints' + description_short: The number of examples provided to train an AI model. Typically, more data results in a more comprehensive understanding by the model. + description_key: + - Training data size refers to the volume of data employed to train an artificial intelligence (AI) model effectively. It's a representation of the number of examples that the model learns from during its training process. It is a fundamental measure of the scope of the data used in the model's learning phase. + + - To grasp the concept of training data size, imagine teaching a friend the art of distinguishing different types of birds. In this analogy, each bird picture presented to your friend corresponds to an individual piece of training data. If you showed them 100 unique bird photos, then the training data size in this scenario would be quantified as 100. + + - Training data size is an essential indicator in AI and machine learning. First and foremost, it directly impacts the depth of learning achieved by the model. The more extensive the dataset, the more profound and comprehensive the model's understanding of the subject matter becomes. Additionally, a large training data size contributes significantly to improved recognition capabilities. By exposing the model to a diverse array of examples, it becomes adept at identifying subtle nuances, much like how it becomes skilled at distinguishing various bird species through exposure to a large variety of bird images. + + display: + numDecimalPlaces: 0 + zeroDay: '1949-01-01' + yearIsDay: true + + training_computation_petaflop: + title: Training computation (petaFLOP) + unit: 'petaFLOP' + description_short: Computation is measured in total petaFLOP, which is 10¹⁵ [floating-point operations](#dod:flop) estimated from AI literature, albeit with some uncertainty. + description_key: + - In the context of artificial intelligence (AI), training computation is predominantly measured using floating-point operations or “FLOP”. One FLOP represents a single arithmetic operation involving floating-point numbers, such as addition, subtraction, multiplication, or division. To adapt to the vast computational demands of AI systems, the measurement unit of petaFLOP is commonly used. One petaFLOP stands as a staggering one quadrillion FLOPs, underscoring the magnitude of computational operations within AI. + + - Modern AI systems are rooted in machine learning and deep learning techniques. These methodologies are notorious for their computational intensity, involving complex mathematical processes and algorithms. During the training phase, AI models process large volumes of data, while continuously adapting and refining their parameters to optimize performance, rendering the training process computationally intensive. + + - Many factors influence the magnitude of training computation within AI systems. Notably, the size of the dataset employed for training significantly impacts the computational load. Larger datasets necessitate more processing power. The complexity of the model's architecture also plays a pivotal role; more intricate models lead to more computations. Parallel processing, involving the simultaneous use of multiple processors, also has a substantial effect. Beyond these factors, specific design choices and other variables further contribute to the complexity and scale of training computation within AI. + + description_processing: Training computation was converted from its original measurement in FLOPs (floating-point operations) to a more manageable unit known as petaFLOPs. This conversion is performed by dividing the original training compute value by 1e15, which represents one quadrillion (10^15). The purpose of this conversion is to provide a more human-readable and practical representation of the immense computational efforts involved in training AI systems. By expressing the training computation in petaFLOPs, it becomes easier to grasp the scale and magnitude of the computational resources required for training these systems, especially when dealing with large datasets and complex architectures. + display: + numDecimalPlaces: 0 + zeroDay: '1949-01-01' + yearIsDay: true + presentation: + grapher_config: + title: Training computation + + publication_date: + title: Publication date + unit: '' + description_short: The date when the AI model was first published. + description_from_producer: The publication, announcement, or release date of the model, in YYYY-MM-DD format. If the year and month are known but the day is unknown, the day is filled in as YYYY-MM-15. If the year is known but the month and day are unknown, the month and day are filled in as YYYY-07-01. + + + diff --git a/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch.py b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch.py new file mode 100644 index 00000000000..1f489c23c58 --- /dev/null +++ b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch.py @@ -0,0 +1,144 @@ +"""Load a meadow dataset and create a garden dataset.""" + +import pandas as pd + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + paths.log.info("epoch.start") + + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("epoch") + + # Read table from meadow dataset. + tb = ds_meadow["epoch"] + tb = tb.reset_index() + + # + # Process data. + # + # Filter notable systems by selecting rows where 'notability_criteria' is not nan + tb = tb[tb["notability_criteria"].notna()].reset_index(drop=True) + tb = tb.drop("notability_criteria", axis=1) + + # Convert relevant columns to string type + columns = ["model", "domain", "organization_categorization"] + tb[columns] = tb[columns].astype(str) + + def simplify_entry(entry): + """ + Simplifies an entry of organization categories which can include many entries of Industry, Academia etc. + Removes duplicates, ensures all words except the first one start with a lower case letter,and joins the categories with ", " and " and " before the last one. + """ + # Check for "nan" + if entry == "nan": + return "Not specified" + + # Split the entry into categories, convert to set to remove duplicates + categories = sorted(set(entry.split(","))) + + # Make sure all words except the first one start with a lower case letter + categories = [categories[0]] + [category.lower() for category in categories[1:]] + + # Join the categories with ", " and " and " before the last one + if len(categories) > 1: + simplified_entry = ", ".join(categories[:-1]) + " and " + categories[-1] + " collaboration" + else: + simplified_entry = categories[0] + + return simplified_entry + + tb["organization_categorization"] = tb["organization_categorization"].apply(simplify_entry) + + # Get the unique values in the organization_categorization column and compare them to expected affiliations + unique_values = set(tb["organization_categorization"]) + expected_values = { + "Industry", + "Academia", + "Government", + "Academia and industry collaboration", + "Academia and research collective collaboration", + "Industry and research collective collaboration", + "Academia, industry and research collective collaboration", + "Government and industry collaboration", + "Research collective", + "Academia, government and industry collaboration", + "Academia and government collaboration", + "Academia, government, industry and research collective collaboration", + "Not specified", + } + assert unique_values == expected_values, "Unexpected affiliations in organization_categorization column" + + # Replace affiliation of researchers with less than 20 systems with 'Other' + affiliation_counts = tb["organization_categorization"].value_counts() + + tb["organization_categorization"] = tb["organization_categorization"].where( + tb["organization_categorization"].map(affiliation_counts) >= 20, "Other" + ) + # Get the organizations that were reclassified to 'Other' + reclassified_organizations = affiliation_counts[affiliation_counts < 20].index.tolist() + + paths.log.info( + f"Affiliations of researchers with less than 20 notable systems that were reclassified to 'Other': {', '.join(reclassified_organizations)}" + ) + + # Replace nans with Unspecified in each column to avoid issues when calculating sume of notable systems + columns = ["organization_categorization", "domain", "organization"] + tb[columns] = tb[columns].replace("nan", "Not specified") + + # Check for multiple entries in 'domain' separated by comma + multiple_domains = tb["domain"].str.contains(",") + # Replace entries in 'domain' that contain a comma with 'Multiple Domains' + tb.loc[multiple_domains, "domain"] = "Multiple domains" + + # Replace domains with less than 20 systems with 'Other' + domain_counts = tb["domain"].value_counts() + + tb["domain"] = tb["domain"].where(tb["domain"].map(domain_counts) >= 20, "Other") + # Get the domains that were reclassified to 'Other' + reclassified_domains = domain_counts[domain_counts < 20].index.tolist() + + paths.log.info( + f"Domains with less than 20 notable systems that were reclassified to 'Other': {', '.join(reclassified_domains)}" + ) + # Convert FLOP to petaFLOP and remove the column with FLOPs (along with training time in hours) + tb["training_computation_petaflop"] = tb["training_compute__flop"] / 1e15 + + # Convert publication date to a datetime objects + tb["publication_date"] = pd.to_datetime(tb["publication_date"]) + + # Calculate 'days_since_1949' + tb["days_since_1949"] = (tb["publication_date"] - pd.to_datetime("1949-01-01")).dt.days.astype("Int64") + tb = tb.dropna(subset=["days_since_1949"]) + + tb = tb.reset_index(drop=True) + + assert not tb[["model", "days_since_1949"]].isnull().any().any(), "Index columns should not have NaN values" + + # Drop columns that are not needed + tb = tb.drop( + ["training_compute__flop", "organization", "authors", "country__from_organization"], + axis=1, + ) + tb = tb.format(["days_since_1949", "model"]) + + # Add metadata to the publication date column + tb["publication_date"].metadata.origins = tb["domain"].metadata.origins + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset(dest_dir, tables=[tb], default_metadata=ds_meadow.metadata) + + # Save changes in the new garden dataset. + ds_garden.save() + + paths.log.info("epoch.end") diff --git a/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_aggregates_affiliation.meta.yml b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_aggregates_affiliation.meta.yml new file mode 100644 index 00000000000..1bf9422d84a --- /dev/null +++ b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_aggregates_affiliation.meta.yml @@ -0,0 +1,35 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + desc_update: The 2024 data is incomplete and was last updated {date_accessed}. + common: + processing_level: major + presentation: + topic_tags: + - Artificial Intelligence + description_short: Describes the sector where the authors of a notable AI system have their primary affiliations. {definitions.desc_update} + description_from_producer: |- + The distinction is documented in [Academia and Industry](https://docs.google.com/document/d/1wyJmDOWDEKItg0QhO5cpsNAgHq4aHOxQQZnTfzm34gI/edit). + Systems are categorized as “Industry” if their authors are affiliated with private sector organizations, “Academia” if the authors are affiliated with universities or academic institutions, or “Industry - Academia Collaboration” when at least 30% of the authors are from each. + Possible values: Industry, Research Collective, Academia, Industry - Academia Collaboration (Industry leaning), Industry - Academia Collaboration (Academia leaning), Non-profit + unit: 'AI systems' + short_unit: '' + display: + numDecimalPlaces: 0 + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/dataset/ +dataset: + update_period_days: 31 + title: Notable AI systems by researcher affiliation + + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/tables/ +tables: + epoch_aggregates_affiliation: + variables: + yearly_count: + title: Annual number of AI systems by researcher affiliation + + cumulative_count: + title: Cumulative number of AI systems by researcher affiliation diff --git a/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_aggregates_affiliation.py b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_aggregates_affiliation.py new file mode 100644 index 00000000000..7bcbf76a4d8 --- /dev/null +++ b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_aggregates_affiliation.py @@ -0,0 +1,75 @@ +"""Generate aggregated table for total yearly and cumulative number of notable AI systems in each category of researcher affiliation.""" + +import datetime as dt + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + paths.log.info("epoch_aggregates_affiliation.start") + + # + # Load inputs. + # + # Load the the garden dataset without aggregations. + ds_garden = paths.load_dataset("epoch") + + # Read table from meadow dataset. + tb = ds_garden["epoch"] + tb = tb.reset_index() + + # + # Process data. + # + # Store the origins metadata for later use + origins = tb["organization_categorization"].metadata.origins + + # Define the columns that are not needed + unused_columns = [ + "days_since_1949", + "parameters", + "training_dataset_size__datapoints", + "domain", + "training_computation_petaflop", + ] + # Drop the unused columns + tb = tb.drop(unused_columns, axis=1) + + # Ensure 'publication_date' column type is datetime64 + assert tb["publication_date"].dtype == "datetime64[ns]", "publication_date column is not of type datetime64" + + # Extract the year from the 'publication_date' column + tb["year"] = tb["publication_date"].dt.year + + # Group by year and country and count the number of systems + tb_agg = tb.groupby(["year", "organization_categorization"], observed=False).size().reset_index(name="yearly_count") + + # Calculate the cumulative count + tb_agg["cumulative_count"] = tb_agg.groupby("organization_categorization", observed=False)["yearly_count"].cumsum() + + # Add the origins metadata to the columns + for col in ["yearly_count", "cumulative_count"]: + tb_agg[col].metadata.origins = origins + + # Set the short_name metadata of the table + tb_agg.metadata.short_name = paths.short_name + + # Set the index to year and country + tb_agg = tb_agg.format(["year", "organization_categorization"]) + + date_acessed = tb_agg.yearly_count.m.origins[0].date_accessed + + # + # Save outputs. + # + ds_garden = create_dataset( + dest_dir, + tables=[tb_agg], + yaml_params={"date_accessed": dt.datetime.strptime(date_acessed, "%Y-%m-%d").strftime("%d %B %Y")}, + ) + ds_garden.save() + + paths.log.info("epoch_aggregates_affiliation.end") diff --git a/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_aggregates_domain.meta.yml b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_aggregates_domain.meta.yml new file mode 100644 index 00000000000..b1cde5bb5d5 --- /dev/null +++ b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_aggregates_domain.meta.yml @@ -0,0 +1,53 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + desc_update: The 2024 data is incomplete and was last updated {date_accessed}. + common: + processing_level: major + presentation: + topic_tags: + - Artificial Intelligence + description_short: Describes the specific area, application, or field in which an AI system is designed to operate. An AI system can operate in more than one domain, thus contributing to the count for multiple domains. {definitions.desc_update} + description_key: + - Game systems are specifically designed for games and excel in understanding and strategizing gameplay. For instance, AlphaGo, developed by DeepMind, defeated the world champion in the game of Go. Such systems use complex algorithms to compete effectively, even against skilled human players. + + - Language systems are tailored to process language, focusing on understanding, translating, and interacting with human languages. Examples include chatbots, machine translation tools like Google Translate, and sentiment analysis algorithms that can detect emotions in text. + + - Multimodal systems are artificial intelligence frameworks that integrate and interpret more than one type of data input, such as text, images, and audio. ChatGPT-4 is an example of a multimodal system, as it has the capability to process and generate responses based on both textual and visual inputs. + + - Vision systems focus on processing visual information, playing a pivotal role in image recognition and related areas. For example, Facebook's photo tagging system uses vision AI to identify faces. + + - Speech systems are dedicated to handling spoken language, serving as the backbone of voice assistants and similar applications. They recognize, interpret, and generate spoken language to interact with users. + + - Recommendation systems offer suggestions based on user preferences, prominently seen in online shopping and media streaming. For instance, Netflix's movie suggestions or Amazon's product recommendations are powered by algorithms that analyze users' preferences and past behaviors. + + - Audio systems process and generate sound, with applications in music composition, signal processing, and sound recognition. + + - Biology systems analyze biological data and simulate biological processes, aiding in drug discovery and genetic research. + + - Image generation systems create visual content from text descriptions or other inputs, used in graphic design and content creation. + + - Robotics systems combine AI with mechanical engineering to create autonomous robots for various industries. + + - Video systems analyze and generate video content, aiding in editing, surveillance, and content creation. + description_processing: The count of notable AI systems per domain is derived by tallying the instances of machine learning models classified under each domain category. It's important to note that a single machine learning model can fall under multiple domains. The classification into domains is determined by the specific area, application, or field that the AI system is primarily designed to operate within. System domains with less than 10 systems are grouped under "Other." + description_from_producer: A foreign key field categorizing the system’s domain of machine learning. This field links to the [ML Domains table](https://airtable.com/appDFXXgaG1xLtXGL/shrhzolGiQCVnwOY5/tbleYEsZORsiYRVTM), and domains are selected from the options in that table. + unit: 'AI systems' + short_unit: '' + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/dataset/ +dataset: + update_period_days: 31 + title: Notable AI systems by domain type + + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/tables/ +tables: + epoch_aggregates_domain: + variables: + yearly_count: + title: Annual number of AI systems by domain + + cumulative_count: + title: Cumulative number of AI systems by domain diff --git a/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_aggregates_domain.py b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_aggregates_domain.py new file mode 100644 index 00000000000..2a4e84e2673 --- /dev/null +++ b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_aggregates_domain.py @@ -0,0 +1,107 @@ +"""Generate aggregated table for total yearly and cumulative number of notable AI systems for each domain.""" + +import datetime as dt + +import pandas as pd + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + paths.log.info("epoch_aggregates_domain.start") + + # + # Load inputs. + # + # Load the ds_meadow dataset. + ds_meadow = paths.load_dataset("epoch") + + # Read table from meadow dataset. + tb = ds_meadow["epoch"] + tb = tb.reset_index() + + # + # Process data. + # + + # Store the origins metadata for later use + origins = tb["domain"].metadata.origins + + # Select the rows where the 'notability_criteria' column is not null (only consider notable systems) + tb = tb[tb["notability_criteria"].notna()].reset_index(drop=True) + + # Define the columns that are not needed + unused_columns = [ + "authors", + "country__from_organization", + "organization", + "organization_categorization", + "parameters", + "training_compute__flop", + "training_dataset_size__datapoints", + "notability_criteria", + ] + # Drop the unused columns + tb = tb.drop(unused_columns, axis=1) + + # Convert the 'publication_date' column to datetime format and extract the year + tb["publication_date"] = pd.to_datetime(tb["publication_date"]) + tb["year"] = tb["publication_date"].dt.year + + # Split the column to be aggregated by comma (several countries/domains can exist in each cell) + tb["domain"] = tb["domain"].str.split(",") + + # Explode the table to create separate rows for each country or domain + tb_exploded = tb.explode("domain") + + # Drop duplicates where the year, model and country/domain are the same + tb_unique = tb_exploded.drop_duplicates(subset=["year", "model", "domain"]) + + # Replace domains with less than 10 systems with 'Other' + domain_counts = tb_unique["domain"].value_counts() + + tb_unique["domain"] = tb_unique["domain"].where(tb_unique["domain"].map(domain_counts) >= 10, "Other") + # Get the domains that were reclassified to 'Other' + reclassified_domains = domain_counts[domain_counts < 10].index.tolist() + domain_counts = tb_unique["domain"].value_counts() + + paths.log.info( + f"Domains with less than 10 notable systems that were reclassified to 'Other': {', '.join(reclassified_domains)}" + ) + # Convert the column to category type so that the missing values will be considered as 0 + tb_unique["domain"] = tb_unique["domain"].astype("category") + + # Group by year and country/domain and count the number of systems (consider all categories which will assume 0 for missing values) + tb_agg = tb_unique.groupby(["year", "domain"], observed=False).size().reset_index(name="yearly_count") + + # Calculate the cumulative count (consider all categories which will assume 0 for missing values) + tb_agg["cumulative_count"] = tb_agg.groupby("domain", observed=False)["yearly_count"].cumsum() + + # Add the origins metadata to the columns + for col in ["yearly_count", "cumulative_count"]: + tb_agg[col].metadata.origins = origins + + # Set the short_name metadata of the table + tb_agg.metadata.short_name = paths.short_name + # Set the index to year and domain + tb_agg = tb_agg.format(["year", "domain"]) + + date_acessed = tb_agg.yearly_count.m.origins[0].date_accessed + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, + tables=[tb_agg], + yaml_params={"date_accessed": dt.datetime.strptime(date_acessed, "%Y-%m-%d").strftime("%d %B %Y")}, + ) + + # Save changes in the new garden dataset. + ds_garden.save() + + paths.log.info("epoch_aggregates_domain.end") diff --git a/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive.meta.yml b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive.meta.yml new file mode 100644 index 00000000000..5f95f506c67 --- /dev/null +++ b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive.meta.yml @@ -0,0 +1,91 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + processing_level: major + presentation: + topic_tags: + - Artificial Intelligence + grapher_config: + note: Confirmed large-scale AI models are those where the training compute exceeds 10²³ floating-point operations (FLOP). +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/dataset/ +dataset: + update_period_days: 31 + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/tables/ +tables: + epoch_compute_intensive: + variables: + domain: + title: Domain + unit: '' + short_unit: '' + description_short: Refers to the specific area, application, or field in which an AI model is designed to operate. + display: + zeroDay: '1949-01-01' + yearIsDay: true + + + parameters: + title: Number of parameters + unit: '' + description_short: Total number of learnable variables or weights that the model contains. Parameters are adjusted during the training process to optimize the model's performance. + description_key: + - Parameters are internal variables that machine learning models adjust during their training process to improve their ability to make accurate predictions. They act as the model's "knobs" that are fine-tuned based on the provided data. In deep learning, a subset of artificial intelligence (AI), parameters primarily consist of the weights assigned to the connections between the small processing units called neurons. Picture a vast network of interconnected neurons where the strength of each connection represents a parameter. + + - The total number of parameters in a model is influenced by various factors. The model's structure and the number of “layers” of neurons play a significant role. Generally, more complex models with additional layers tend to have a higher number of parameters. Special components of specific deep learning architectures can further contribute to the overall parameter count. + + - Understanding the number of parameters in a model is crucial to design effective models. More parameters can help the model understand complex data patterns, potentially leading to higher accuracy. However, there's a fine balance to strike. If a model has too many parameters, it risks memorizing the specific examples in its training data rather than learning their underlying patterns. Consequently, it may perform poorly when presented with new, unseen data. Achieving the right balance of parameters is a critical consideration in model development. + + - In recent times, the AI community has witnessed the emergence of what are often referred to as "giant models." These models boast an astounding number of parameters, reaching into the billions or even trillions. While these huge models have achieved remarkable performance, they have a significant computational cost. Effectively managing and training such large-scale models has become a prominent and active area of research and discussion within the AI field. + + display: + numDecimalPlaces: 0 + zeroDay: '1949-01-01' + yearIsDay: true + + training_dataset_size__datapoints: + title: Training dataset size + unit: 'datapoints' + description_short: The number of examples provided to train an AI model. Typically, more data results in a more comprehensive understanding by the model. + description_key: + - Training data size refers to the volume of data employed to train an artificial intelligence (AI) model effectively. It's a representation of the number of examples that the model learns from during its training process. It is a fundamental measure of the scope of the data used in the model's learning phase. + + - To grasp the concept of training data size, imagine teaching a friend the art of distinguishing different types of birds. In this analogy, each bird picture presented to your friend corresponds to an individual piece of training data. If you showed them 100 unique bird photos, then the training data size in this scenario would be quantified as 100. + + - Training data size is an essential indicator in AI and machine learning. First and foremost, it directly impacts the depth of learning achieved by the model. The more extensive the dataset, the more profound and comprehensive the model's understanding of the subject matter becomes. Additionally, a large training data size contributes significantly to improved recognition capabilities. By exposing the model to a diverse array of examples, it becomes adept at identifying subtle nuances, much like how it becomes skilled at distinguishing various bird species through exposure to a large variety of bird images. + + display: + numDecimalPlaces: 0 + zeroDay: '1949-01-01' + yearIsDay: true + + training_computation_petaflop: + title: Training computation (petaFLOP) + unit: 'petaFLOP' + description_short: Computation is measured in total petaFLOP, which is 10¹⁵ [floating-point operations](#dod:flop) estimated from AI literature, albeit with some uncertainty. + description_key: + - In the context of artificial intelligence (AI), training computation is predominantly measured using floating-point operations or “FLOP”. One FLOP represents a single arithmetic operation involving floating-point numbers, such as addition, subtraction, multiplication, or division. To adapt to the vast computational demands of AI systems, the measurement unit of petaFLOP is commonly used. One petaFLOP stands as a staggering one quadrillion FLOPs, underscoring the magnitude of computational operations within AI. + + - Modern AI systems are rooted in machine learning and deep learning techniques. These methodologies are notorious for their computational intensity, involving complex mathematical processes and algorithms. During the training phase, AI models process large volumes of data, while continuously adapting and refining their parameters to optimize performance, rendering the training process computationally intensive. + + - Many factors influence the magnitude of training computation within AI systems. Notably, the size of the dataset employed for training significantly impacts the computational load. Larger datasets necessitate more processing power. The complexity of the model's architecture also plays a pivotal role; more intricate models lead to more computations. Parallel processing, involving the simultaneous use of multiple processors, also has a substantial effect. Beyond these factors, specific design choices and other variables further contribute to the complexity and scale of training computation within AI. + + description_processing: Training computation was converted from its original measurement in FLOPs (floating-point operations) to a more manageable unit known as petaFLOPs. This conversion is performed by dividing the original training compute value by 1e15, which represents one quadrillion (10^15). The purpose of this conversion is to provide a more human-readable and practical representation of the immense computational efforts involved in training AI systems. By expressing the training computation in petaFLOPs, it becomes easier to grasp the scale and magnitude of the computational resources required for training these systems, especially when dealing with large datasets and complex architectures. + display: + numDecimalPlaces: 0 + zeroDay: '1949-01-01' + yearIsDay: true + presentation: + grapher_config: + title: Training computation + + publication_date: + title: Publication date + unit: '' + description_short: The date when the AI model was first published. + description_from_producer: The publication, announcement, or release date of the model, in YYYY-MM-DD format. If the year and month are known but the day is unknown, the day is filled in as YYYY-MM-15. If the year is known but the month and day are unknown, the month and day are filled in as YYYY-07-01. + + + diff --git a/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive.py b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive.py new file mode 100644 index 00000000000..b9b431c4ef0 --- /dev/null +++ b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive.py @@ -0,0 +1,60 @@ +"""Load a meadow dataset and create a garden dataset.""" + +import pandas as pd + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + paths.log.info("epoch.start") + + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("epoch_compute_intensive") + + # Read table from meadow dataset. + tb = ds_meadow["epoch_compute_intensive"] + tb = tb.reset_index() + + # + # Process data. + # + # Convert FLOP to petaFLOP and remove the column with FLOPs (along with training time in hours) + tb["training_computation_petaflop"] = tb["training_compute__flop"] / 1e15 + + # Convert publication date to a datetime objects + tb["publication_date"] = pd.to_datetime(tb["publication_date"]) + + # Calculate 'days_since_1949' + tb["days_since_1949"] = (tb["publication_date"] - pd.to_datetime("1949-01-01")).dt.days.astype("Int64") + tb = tb.dropna(subset=["days_since_1949"]) + + tb = tb.reset_index(drop=True) + + assert not tb[["model", "days_since_1949"]].isnull().any().any(), "Index columns should not have NaN values" + + # Drop columns that are not needed + tb = tb.drop( + ["training_compute__flop", "organization", "authors", "country__from_organization"], + axis=1, + ) + tb = tb.format(["days_since_1949", "model"]) + + # Add metadata to the publication date column + tb["publication_date"].metadata.origins = tb["domain"].metadata.origins + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset(dest_dir, tables=[tb], default_metadata=ds_meadow.metadata) + + # Save changes in the new garden dataset. + ds_garden.save() + + paths.log.info("epoch_compute_intensive.end") diff --git a/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive_countries.countries.json b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive_countries.countries.json new file mode 100644 index 00000000000..ddfda66807a --- /dev/null +++ b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive_countries.countries.json @@ -0,0 +1,18 @@ +{ + "Canada": "Canada", + "China": "China", + "Germany": "Germany", + "Israel": "Israel", + "Singapore": "Singapore", + "United Arab Emirates": "United Arab Emirates", + "United Kingdom of Great Britain and Northern Ireland": "United Kingdom", + "United States of America": "United States", + "Korea (Republic of)": "South Korea", + "Multinational": "Multinational", + "Russia": "Russia", + "Japan": "Japan", + "France": "France", + "Finland": "Finland", + "Total": "Total", + "Hong Kong": "Hong Kong" +} \ No newline at end of file diff --git a/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive_countries.meta.yml b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive_countries.meta.yml new file mode 100644 index 00000000000..3f97637a89b --- /dev/null +++ b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive_countries.meta.yml @@ -0,0 +1,31 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + desc_update: The 2024 data is incomplete and was last updated {date_accessed}. + common: + processing_level: major + presentation: + topic_tags: + - Artificial Intelligence + grapher_config: + note: Confirmed large-scale AI models are those where the training compute exceeds 10²³ floating-point operations (FLOP). + + unit: 'AI systems' + short_unit: '' + description_short: Refers to the location of the primary organization with which the authors of a large-scale AI systems are affiliated. {definitions.desc_update} + description_processing: The number of large-scale AI systems by country is determined by tallying the number of machine learning models that are associated with the geographical location of the researchers' affiliated institutions. It's important to note that a single model can have multiple authors, each potentially affiliated with different institutions, thus contributing to the count for multiple countries. +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/dataset/ + +dataset: + update_period_days: 31 + title: Large-scale AI systems by country +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/tables/ +tables: + epoch_compute_intensive_countries: + variables: + yearly_count: + title: Annual number of large-scale AI systems by country + + cumulative_count: + title: Cumulative number of large-scale AI systems by country diff --git a/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive_countries.py b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive_countries.py new file mode 100644 index 00000000000..69bc951a631 --- /dev/null +++ b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive_countries.py @@ -0,0 +1,67 @@ +"""Generate aggregated table for total yearly and cumulative number of compute intensive AI systems in each country.""" + +import datetime as dt + +import shared as sh + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + paths.log.info("epoch_compute_intensive_countries.start") + + # + # Load inputs. + # + # Load the ds_meadow dataset. + ds_meadow = paths.load_dataset("epoch_compute_intensive") + + # Read table from meadow dataset. + tb = ds_meadow["epoch_compute_intensive"] + tb = tb.reset_index() + + # + # Process data. + # + # Define the columns that are not needed + unused_columns = [ + "domain", + "authors", + "organization", + "parameters", + "training_compute__flop", + "training_dataset_size__datapoints", + ] + + # Aggregate the data by country + tb_agg = sh.calculate_aggregates(tb, "country__from_organization", paths.short_name, unused_columns) + + # Rename the 'country__from_organization' column to 'country' + tb_agg = tb_agg.rename(columns={"country__from_organization": "country"}) + + # Harmonize the country names + tb_agg = geo.harmonize_countries(df=tb_agg, countries_file=paths.country_mapping_path) + + # Set the index to year and country + tb_agg = tb_agg.format(["year", "country"]) + + date_acessed = tb_agg.yearly_count.m.origins[0].date_accessed + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, + tables=[tb_agg], + yaml_params={"date_accessed": dt.datetime.strptime(date_acessed, "%Y-%m-%d").strftime("%d %B %Y")}, + ) + + # Save changes in the new garden dataset. + ds_garden.save() + + paths.log.info("epoch_compute_intensive_countries.end") diff --git a/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive_domain.meta.yml b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive_domain.meta.yml new file mode 100644 index 00000000000..4d6697e7541 --- /dev/null +++ b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive_domain.meta.yml @@ -0,0 +1,48 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + desc_update: The 2024 data is incomplete and was last updated {date_accessed}. + common: + processing_level: major + presentation: + topic_tags: + - Artificial Intelligence + grapher_config: + note: Confirmed large-scale AI models are those where the training compute exceeds 10²³ floating-point operations (FLOP). + description_short: Describes the specific area, application, or field in which a large-scale AI model is designed to operate. {definitions.desc_update} + description_key: + - Game systems are specifically designed for games and excel in understanding and strategizing gameplay. For instance, AlphaGo, developed by DeepMind, defeated the world champion in the game of Go. Such systems use complex algorithms to compete effectively, even against skilled human players. + + - Language systems are tailored to process language, focusing on understanding, translating, and interacting with human languages. Examples include chatbots, machine translation tools like Google Translate, and sentiment analysis algorithms that can detect emotions in text. + + - Multimodal systems are artificial intelligence frameworks that integrate and interpret more than one type of data input, such as text, images, and audio. ChatGPT-4 is an example of a multimodal model, as it has the capability to process and generate responses based on both textual and visual inputs. + + - Vision systems focus on processing visual information, playing a pivotal role in image recognition and related areas. For example, Facebook's photo tagging model uses vision AI to identify faces. + + - Speech systems are dedicated to handling spoken language, serving as the backbone of voice assistants and similar applications. They recognize, interpret, and generate spoken language to interact with users. + + - Biology systems analyze biological data and simulate biological processes, aiding in drug discovery and genetic research. + + - Image generation systems create visual content from text descriptions or other inputs, used in graphic design and content creation. + + description_processing: The count of large-scale AI models AI systems per domain is derived by tallying the instances of machine learning models classified under each domain category. It's important to note that a single machine learning model can fall under multiple domains. The classification into domains is determined by the specific area, application, or field that the AI model is primarily designed to operate within. + description_from_producer: A foreign key field categorizing the system’s domain of machine learning. This field links to the [ML Domains table](https://airtable.com/appDFXXgaG1xLtXGL/shrhzolGiQCVnwOY5/tbleYEsZORsiYRVTM), and domains are selected from the options in that table. + unit: 'AI systems' + short_unit: '' + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/dataset/ +dataset: + update_period_days: 31 + title: Large-scale AI systems by domain type + + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/tables/ +tables: + epoch_compute_intensive_domain: + variables: + yearly_count: + title: Annual number of large-scale AI models by domain + + cumulative_count: + title: Cumulative number of large-scale AI models by domain diff --git a/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive_domain.py b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive_domain.py new file mode 100644 index 00000000000..e832677a43d --- /dev/null +++ b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_compute_intensive_domain.py @@ -0,0 +1,60 @@ +"""Generate aggregated table for total yearly and cumulative number of compute intensive AI systems for each domain.""" + +import datetime as dt + +import shared as sh + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + paths.log.info("epoch_compute_intensive_domain.start") + + # + # Load inputs. + # + # Load the ds_meadow dataset. + ds_meadow = paths.load_dataset("epoch_compute_intensive") + + # Read table from meadow dataset. + tb = ds_meadow["epoch_compute_intensive"] + tb = tb.reset_index() + + # + # Process data. + # + # Define the columns that are not needed + unused_columns = [ + "authors", + "country__from_organization", + "organization", + "parameters", + "training_compute__flop", + "training_dataset_size__datapoints", + ] + + # Aggregate the data by domain + tb_agg = sh.calculate_aggregates(tb, "domain", paths.short_name, unused_columns) + + # Set the index to year and domain + tb_agg = tb_agg.format(["year", "domain"]) + + date_acessed = tb_agg.yearly_count.m.origins[0].date_accessed + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, + tables=[tb_agg], + yaml_params={"date_accessed": dt.datetime.strptime(date_acessed, "%Y-%m-%d").strftime("%d %B %Y")}, + ) + + # Save changes in the new garden dataset. + ds_garden.save() + + paths.log.info("epoch_compute_intensive_domain.end") diff --git a/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_regressions.meta.yml b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_regressions.meta.yml new file mode 100644 index 00000000000..8bffd4fdf09 --- /dev/null +++ b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_regressions.meta.yml @@ -0,0 +1,12 @@ + +definitions: + common: + processing_level: major + presentation: + topic_tags: + - Artificial Intelligence + description_processing: |- + We performed a regression analysis, fitting exponential models to the data for both the pre-deep learning (before 2010) and deep learning eras (after 2010), using the code provided by researchers from Epoch. +dataset: + title: Parameter, Compute and Data Trends in Machine Learning - Regressions + diff --git a/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_regressions.py b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_regressions.py new file mode 100644 index 00000000000..8968e2c76a4 --- /dev/null +++ b/etl/steps/data/garden/artificial_intelligence/2024-12-05/epoch_regressions.py @@ -0,0 +1,145 @@ +"""Load a meadow dataset and create a garden dataset.""" + +import numpy as np +import owid.catalog.processing as pr +import pandas as pd +from owid.catalog import Table +from sklearn.linear_model import LinearRegression + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) +# Constants for defining the time periods +DL_ERA_START = 2010 +START_DATE = 1950 +END_DATE = 2025.2 + + +def run(dest_dir: str) -> None: + paths.log.info("epoch.start") + + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("epoch") + + # Read table from meadow dataset. + tb = ds_meadow["epoch"].reset_index() + + # Run regression analysis and concatenate results + tb_trend = run_regression(tb) + tb = tb.drop("frac_year", axis=1) + tb = pr.concat([tb_trend, tb]) + + # Format the table + tb = tb.format(["days_since_1949", "model"]) + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset(dest_dir, tables=[tb], default_metadata=ds_meadow.metadata) + + # Save changes in the new garden dataset. + ds_garden.save() + + paths.log.info("epoch.end") + + +def fit_exponential(models, metric): + """Fit an exponential model to the given metric data. Code provided by Epoch AI team.""" + x = models["frac_year"].values.reshape(-1, 1) + y = models[metric] + + # Filter out non-positive values + positive_mask = y > 0 + x = x[positive_mask] + y = y[positive_mask] + + # Apply log10 transformation + y = np.log10(y) + + # Filter out infinite and extremely large values + finite_mask = np.isfinite(y) & (y < np.finfo(np.float32).max) + x = x[finite_mask] + y = y[finite_mask] + + # Fit linear regression model + reg = LinearRegression().fit(x, y) + return reg.intercept_, reg.coef_[0] + + +def run_regression(tb): + """Run regression analysis on the given table and return the updated table.""" + # Add fractional year for sorting and processing + publication_dates = tb["publication_date"] + tb.loc[:, "frac_year"] = ( + publication_dates.dt.year + (publication_dates.dt.month - 1) / 12 + (publication_dates.dt.day - 1) / 365 + ) + tb = tb.sort_values(by="frac_year") + + # Define periods dynamically + periods = { + f"{START_DATE}–{DL_ERA_START}": (tb["frac_year"] < DL_ERA_START), + f"{DL_ERA_START}–{int(END_DATE)}": ((tb["frac_year"] >= DL_ERA_START) & (tb["frac_year"] < END_DATE)), + } + # Define year grids dynamically + year_grids = { + f"{START_DATE}–{DL_ERA_START}": np.array([START_DATE, DL_ERA_START]), + f"{DL_ERA_START}–{int(END_DATE)}": np.array([DL_ERA_START, END_DATE]), + } + + metrics = ["training_computation_petaflop", "parameters", "training_dataset_size__datapoints"] + new_tables = [] + + for metric in metrics: + # Filter out models without the metric information + tb_metric = tb[pd.notnull(tb[metric])] + dfs = [] + + for period_name, condition in periods.items(): + # Subset data for the current period + period_data = tb_metric[condition] + + # Fit exponential model + fit = fit_exponential(period_data, metric) + oom_per_year = fit[1] + info = f"{10**oom_per_year:.1f}x/year" + + # Log the results + paths.log.info(f"{period_name} ({metric}): {info}") + + # Calculate the regression line for the current period + year_grid = year_grids[period_name] + line = 10 ** (fit[0] + year_grid * fit[1]) + + # Create DataFrame for the current period + df = pd.DataFrame( + { + "days_since_1949": [ + period_data["days_since_1949"].min(), + period_data["days_since_1949"].max(), + ], + f"{metric}": [line[0], line[-1]], + "model": [f"{info} between {period_name}"] * 2, + } + ) + dfs.append(df) + + # Combine the DataFrames for all periods for the current metric + df_combined = pd.concat(dfs, ignore_index=True) + new_tables.append(df_combined) + + # Merge all the new DataFrames + tb_new = new_tables[0] + for tb_m in new_tables[1:]: + tb_new = pd.merge(tb_new, tb_m, on=["model", "days_since_1949"], how="outer") + + # Convert to OWID Table and add metadata + tb_new = Table(tb_new, short_name=paths.short_name) + for column in tb_new.columns: + tb_new[column].metadata.origins = tb["publication_date"].metadata.origins + + return tb_new diff --git a/etl/steps/data/garden/artificial_intelligence/2024-12-05/shared.py b/etl/steps/data/garden/artificial_intelligence/2024-12-05/shared.py new file mode 100644 index 00000000000..016e6812e4d --- /dev/null +++ b/etl/steps/data/garden/artificial_intelligence/2024-12-05/shared.py @@ -0,0 +1,74 @@ +from typing import List + +import owid.catalog.processing as pr +import pandas as pd +from owid.catalog import Table + +from etl.helpers import PathFinder + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def calculate_aggregates(tb: Table, agg_column: str, short_name: str, unused_columns: List[str]) -> Table: + """ + This function calculates aggregates for a given column in a Table. It is used to calculate the total yearly and cumulative number of notable AI systems for each domain or country. + + Parameters: + tb (Table): The input Table. + agg_column (str): The column to aggregate on. + short_name (str): The short name to set for the table. + unused_columns (List[str]): The list of columns to drop from the table. + + Returns: + Table: The output Table with calculated aggregates. + """ + + # Store the origins metadata for later use + origins = tb[agg_column].metadata.origins + + # Drop the unused columns + tb = tb.drop(unused_columns, axis=1) + + # Convert the 'publication_date' column to datetime format and extract the year + tb["publication_date"] = pd.to_datetime(tb["publication_date"]) + tb["year"] = tb["publication_date"].dt.year + + # Convert the column to category type so that the missing values will be considered as 0 + tb[agg_column] = tb[agg_column].astype("category") + + # Group total yearly counts and calculate cumulative count for total number of systems + tb_total = tb.groupby(["year"]).size().reset_index(name="yearly_count") + total_counts = tb_total.groupby("year")["yearly_count"].sum().reset_index() + total_counts[agg_column] = "Total" + total_counts["cumulative_count"] = total_counts["yearly_count"].cumsum() + + # Split the column to be aggregated by comma (several countries/domains can exist in each cell) + tb[agg_column] = tb[agg_column].str.split(",") + + # Explode the table to create separate rows for each country or domain + tb_exploded = tb.explode(agg_column) + + # Convert the column to category type so that the missing values will be considered as 0 + tb_exploded[agg_column] = tb_exploded[agg_column].astype("category") + + # Drop duplicates where the year, model and country/domain are the same + tb_unique = tb_exploded.drop_duplicates(subset=["year", "model", agg_column]) + + # Group by year and country/domain and count the number of systems (consider all categories which will assume 0 for missing values) + tb_agg = tb_unique.groupby(["year", agg_column], observed=False).size().reset_index(name="yearly_count") + + # Calculate the cumulative count (consider all categories which will assume 0 for missing values) + tb_agg["cumulative_count"] = tb_agg.groupby(agg_column, observed=False)["yearly_count"].cumsum() + + # Combine aggregated data with total counts + tb_agg = pr.concat([tb_agg, total_counts], ignore_index=True) + + # Add the origins metadata to the columns + for col in ["yearly_count", "cumulative_count"]: + tb_agg[col].metadata.origins = origins + + # Set the short_name metadata of the table + tb_agg.metadata.short_name = short_name + + return tb_agg diff --git a/etl/steps/data/garden/artificial_intelligence/latest/epoch.py b/etl/steps/data/garden/artificial_intelligence/latest/epoch.py index e5af4577835..d7b6839d5c3 100644 --- a/etl/steps/data/garden/artificial_intelligence/latest/epoch.py +++ b/etl/steps/data/garden/artificial_intelligence/latest/epoch.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + import pandas as pd from structlog import get_logger diff --git a/etl/steps/data/garden/artificial_intelligence/latest/epoch_aggregates_affiliation.py b/etl/steps/data/garden/artificial_intelligence/latest/epoch_aggregates_affiliation.py index d2ffa0738af..52d32304128 100644 --- a/etl/steps/data/garden/artificial_intelligence/latest/epoch_aggregates_affiliation.py +++ b/etl/steps/data/garden/artificial_intelligence/latest/epoch_aggregates_affiliation.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + import owid.catalog.processing as pr from structlog import get_logger diff --git a/etl/steps/data/garden/artificial_intelligence/latest/epoch_aggregates_approach.py b/etl/steps/data/garden/artificial_intelligence/latest/epoch_aggregates_approach.py index 3c588ba3553..b316d8d939c 100644 --- a/etl/steps/data/garden/artificial_intelligence/latest/epoch_aggregates_approach.py +++ b/etl/steps/data/garden/artificial_intelligence/latest/epoch_aggregates_approach.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + import owid.catalog.processing as pr from structlog import get_logger diff --git a/etl/steps/data/garden/artificial_intelligence/latest/epoch_aggregates_domain.py b/etl/steps/data/garden/artificial_intelligence/latest/epoch_aggregates_domain.py index 7acec5dafb9..03d952d68b5 100644 --- a/etl/steps/data/garden/artificial_intelligence/latest/epoch_aggregates_domain.py +++ b/etl/steps/data/garden/artificial_intelligence/latest/epoch_aggregates_domain.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + import owid.catalog.processing as pr from structlog import get_logger diff --git a/etl/steps/data/garden/cancer/2024-10-13/gco_cancer_over_time_cervical.py b/etl/steps/data/garden/cancer/2024-10-13/gco_cancer_over_time_cervical.py index 76f64b2b0a5..a829461174c 100644 --- a/etl/steps/data/garden/cancer/2024-10-13/gco_cancer_over_time_cervical.py +++ b/etl/steps/data/garden/cancer/2024-10-13/gco_cancer_over_time_cervical.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - from etl.data_helpers import geo from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/garden/climate/2024-01-31/climate_change_impacts.py b/etl/steps/data/garden/climate/2024-01-31/climate_change_impacts.py index 38f00ffd808..81d7e86da05 100644 --- a/etl/steps/data/garden/climate/2024-01-31/climate_change_impacts.py +++ b/etl/steps/data/garden/climate/2024-01-31/climate_change_impacts.py @@ -1,6 +1,4 @@ -"""Create a garden dataset with all climate change impacts data. - -""" +"""Create a garden dataset with all climate change impacts data.""" from owid.catalog import Table from owid.datautils.dataframes import combine_two_overlapping_dataframes diff --git a/etl/steps/data/garden/climate/2024-03-11/climate_change_impacts.py b/etl/steps/data/garden/climate/2024-03-11/climate_change_impacts.py index 38f00ffd808..81d7e86da05 100644 --- a/etl/steps/data/garden/climate/2024-03-11/climate_change_impacts.py +++ b/etl/steps/data/garden/climate/2024-03-11/climate_change_impacts.py @@ -1,6 +1,4 @@ -"""Create a garden dataset with all climate change impacts data. - -""" +"""Create a garden dataset with all climate change impacts data.""" from owid.catalog import Table from owid.datautils.dataframes import combine_two_overlapping_dataframes diff --git a/etl/steps/data/garden/climate/2024-04-17/climate_change_impacts.py b/etl/steps/data/garden/climate/2024-04-17/climate_change_impacts.py index 38f00ffd808..81d7e86da05 100644 --- a/etl/steps/data/garden/climate/2024-04-17/climate_change_impacts.py +++ b/etl/steps/data/garden/climate/2024-04-17/climate_change_impacts.py @@ -1,6 +1,4 @@ -"""Create a garden dataset with all climate change impacts data. - -""" +"""Create a garden dataset with all climate change impacts data.""" from owid.catalog import Table from owid.datautils.dataframes import combine_two_overlapping_dataframes diff --git a/etl/steps/data/garden/climate/2024-05-20/climate_change_impacts.py b/etl/steps/data/garden/climate/2024-05-20/climate_change_impacts.py index 38f00ffd808..81d7e86da05 100644 --- a/etl/steps/data/garden/climate/2024-05-20/climate_change_impacts.py +++ b/etl/steps/data/garden/climate/2024-05-20/climate_change_impacts.py @@ -1,6 +1,4 @@ -"""Create a garden dataset with all climate change impacts data. - -""" +"""Create a garden dataset with all climate change impacts data.""" from owid.catalog import Table from owid.datautils.dataframes import combine_two_overlapping_dataframes diff --git a/etl/steps/data/garden/climate/2024-07-23/climate_change_impacts.py b/etl/steps/data/garden/climate/2024-07-23/climate_change_impacts.py index 38f00ffd808..81d7e86da05 100644 --- a/etl/steps/data/garden/climate/2024-07-23/climate_change_impacts.py +++ b/etl/steps/data/garden/climate/2024-07-23/climate_change_impacts.py @@ -1,6 +1,4 @@ -"""Create a garden dataset with all climate change impacts data. - -""" +"""Create a garden dataset with all climate change impacts data.""" from owid.catalog import Table from owid.datautils.dataframes import combine_two_overlapping_dataframes diff --git a/etl/steps/data/garden/climate/2024-09-30/climate_change_impacts.py b/etl/steps/data/garden/climate/2024-09-30/climate_change_impacts.py index 38f00ffd808..81d7e86da05 100644 --- a/etl/steps/data/garden/climate/2024-09-30/climate_change_impacts.py +++ b/etl/steps/data/garden/climate/2024-09-30/climate_change_impacts.py @@ -1,6 +1,4 @@ -"""Create a garden dataset with all climate change impacts data. - -""" +"""Create a garden dataset with all climate change impacts data.""" from owid.catalog import Table from owid.datautils.dataframes import combine_two_overlapping_dataframes diff --git a/etl/steps/data/garden/climate/2024-11-18/climate_change_impacts.py b/etl/steps/data/garden/climate/2024-11-18/climate_change_impacts.py index 38f00ffd808..81d7e86da05 100644 --- a/etl/steps/data/garden/climate/2024-11-18/climate_change_impacts.py +++ b/etl/steps/data/garden/climate/2024-11-18/climate_change_impacts.py @@ -1,6 +1,4 @@ -"""Create a garden dataset with all climate change impacts data. - -""" +"""Create a garden dataset with all climate change impacts data.""" from owid.catalog import Table from owid.datautils.dataframes import combine_two_overlapping_dataframes diff --git a/etl/steps/data/garden/climate/latest/weekly_wildfires.py b/etl/steps/data/garden/climate/latest/weekly_wildfires.py index cb0f80f5c68..d9760f9d99c 100644 --- a/etl/steps/data/garden/climate/latest/weekly_wildfires.py +++ b/etl/steps/data/garden/climate/latest/weekly_wildfires.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - import owid.catalog.processing as pr import pandas as pd diff --git a/etl/steps/data/garden/countries/2024-08-27/gleditsch.py b/etl/steps/data/garden/countries/2024-08-27/gleditsch.py index a18d6b14e4b..66b05096cdf 100644 --- a/etl/steps/data/garden/countries/2024-08-27/gleditsch.py +++ b/etl/steps/data/garden/countries/2024-08-27/gleditsch.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + from datetime import datetime as dt import owid.catalog.processing as pr diff --git a/etl/steps/data/garden/covid/latest/compact.py b/etl/steps/data/garden/covid/latest/compact.py index 25f50a695da..fe684152bbe 100644 --- a/etl/steps/data/garden/covid/latest/compact.py +++ b/etl/steps/data/garden/covid/latest/compact.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + from ast import literal_eval from typing import cast diff --git a/etl/steps/data/garden/covid/latest/countries_reporting.meta.yml b/etl/steps/data/garden/covid/latest/countries_reporting.meta.yml index c9cc581a0ec..3ab26130309 100644 --- a/etl/steps/data/garden/covid/latest/countries_reporting.meta.yml +++ b/etl/steps/data/garden/covid/latest/countries_reporting.meta.yml @@ -71,9 +71,9 @@ tables: country_reporting_delay: variables: num_days_delay_in_reporting: - title: Number of days needed to first report data on COVID-19 << type >> since the first vaccine was administered + title: Number of days needed to first report data on COVID-19 << type | default('') >> since the first vaccine was administered description_short: |- - Number of days needed to first report data on COVID-19 << type >> since the first vaccine was administered. Some countries may have started vaccinating before they reported it, or may have started reporting it before they started vaccinating. + Number of days needed to first report data on COVID-19 << type | default('') >> since the first vaccine was administered. Some countries may have started vaccinating before they reported it, or may have started reporting it before they started vaccinating. description_processing: *processing_2 description_key: *description_key_2 unit: "days" diff --git a/etl/steps/data/garden/covid/latest/countries_reporting.py b/etl/steps/data/garden/covid/latest/countries_reporting.py index 3742ff433a6..af37073ba11 100644 --- a/etl/steps/data/garden/covid/latest/countries_reporting.py +++ b/etl/steps/data/garden/covid/latest/countries_reporting.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - from owid.catalog import Table from etl.data_helpers.misc import expand_time_column diff --git a/etl/steps/data/garden/covid/latest/infections_model.py b/etl/steps/data/garden/covid/latest/infections_model.py index 2ce0b1c7a25..891e80d3731 100644 --- a/etl/steps/data/garden/covid/latest/infections_model.py +++ b/etl/steps/data/garden/covid/latest/infections_model.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + from typing import Any, Dict from owid.catalog import Table diff --git a/etl/steps/data/garden/covid/latest/sequence.py b/etl/steps/data/garden/covid/latest/sequence.py index e0feee79222..4061b125180 100644 --- a/etl/steps/data/garden/covid/latest/sequence.py +++ b/etl/steps/data/garden/covid/latest/sequence.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + from datetime import timedelta import owid.catalog.processing as pr diff --git a/etl/steps/data/garden/covid/latest/sweden_covid.py b/etl/steps/data/garden/covid/latest/sweden_covid.py index 6e5e6da2a7a..940f7946c72 100644 --- a/etl/steps/data/garden/covid/latest/sweden_covid.py +++ b/etl/steps/data/garden/covid/latest/sweden_covid.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + import numpy as np from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/garden/covid/latest/vaccinations_us.meta.yml b/etl/steps/data/garden/covid/latest/vaccinations_us.meta.yml index 94d5622f023..f987a11471a 100644 --- a/etl/steps/data/garden/covid/latest/vaccinations_us.meta.yml +++ b/etl/steps/data/garden/covid/latest/vaccinations_us.meta.yml @@ -1,29 +1,21 @@ # NOTE: To learn more about the fields, hover over their names. definitions: - display_zero_day: &display_zero_day - zeroDay: 2020-01-01 - yearIsDay: true common: - display: - numDecimalPlaces: 0 - <<: *display_zero_day + numDecimalPlaces: 0 presentation: topic_tags: - COVID-19 - # Learn more about the available fields: # http://docs.owid.io/projects/etl/architecture/metadata/reference/ dataset: update_period_days: 365 - title: COVID-19, Vaccinations (United States) tables: vaccinations_us: variables: - total_vaccinations: title: Total vaccinations unit: doses @@ -31,85 +23,77 @@ tables: description_processing: All doses, including boosters, are counted individually. total_vaccinations_per_hundred: title: Total vaccinations (per 100 people) - unit: 'doses per 100 people' + unit: "doses per 100 people" description_short: Cumularive number of COVID-19 vaccination doses administered, per 100 people. description_processing: All doses, including boosters, are counted individually. display: numDecimalPlaces: 2 - <<: *display_zero_day total_distributed: title: Total doses distributed - unit: 'doses' + unit: "doses" description_short: Cumulative counts of COVID-19 vaccine doses reported to Operation Warp Speed as delivered. distributed_per_hundred: title: Total doses distributed (per 100 people) - unit: 'doses per 100 people' + unit: "doses per 100 people" description_short: Cumulative counts of COVID-19 vaccine doses reported to Operation Warp Speed as delivered, per 100 people. display: numDecimalPlaces: 2 - <<: *display_zero_day people_vaccinated: title: People vaccinated description_short: Total number of people who received at least one vaccine dose. - unit: 'people' + unit: "people" people_vaccinated_per_hundred: title: People vaccinated (per 100 people) description_short: Share of people who received at least one vaccine dose. - unit: '%' - short_unit: '%' + unit: "%" + short_unit: "%" display: numDecimalPlaces: 2 - <<: *display_zero_day people_fully_vaccinated: title: People fully vaccinated description_short: Total number of people who received all doses prescribed by the vaccination protocol. - unit: 'people' + unit: "people" people_fully_vaccinated_per_hundred: title: People fully vaccinated (per 100 people) description_short: Share of people who received all doses prescribed by the vaccination protocol. - unit: '%' - short_unit: '%' + unit: "%" + short_unit: "%" display: numDecimalPlaces: 2 - <<: *display_zero_day total_boosters: title: Total booster doses administered - unit: 'doses' + unit: "doses" total_boosters_per_hundred: title: Total booster doses administered (per 100 people) - unit: 'doses per 100 people' + unit: "doses per 100 people" display: numDecimalPlaces: 3 - <<: *display_zero_day daily_vaccinations: title: Daily doses administered (7-day rolling average) description_short: All doses, including boosters, are counted individually. 7-day rolling average. - unit: 'doses' + unit: "doses" display: numDecimalPlaces: 2 - <<: *display_zero_day daily_vaccinations_raw: title: Daily doses administered description_short: All doses, including boosters, are counted individually. - unit: 'doses' + unit: "doses" daily_vaccinations_per_million: title: Daily doses administered (per million people) description_short: All doses, including boosters, are counted individually, per million people - unit: 'doses per million people' + unit: "doses per million people" display: numDecimalPlaces: 2 - <<: *display_zero_day share_doses_used: title: Share of doses used description_short: Share of distributed vaccination doses that have been administered/used in the population. Distributed figures represent those reported to Operation Warp Speed as delivered. - unit: '%' - short_unit: '%' + unit: "%" + short_unit: "%" display: numDecimalPlaces: 1 - <<: *display_zero_day diff --git a/etl/steps/data/garden/covid/latest/yougov.meta.yml b/etl/steps/data/garden/covid/latest/yougov.meta.yml index d4ac7be9e20..03e09284fd1 100644 --- a/etl/steps/data/garden/covid/latest/yougov.meta.yml +++ b/etl/steps/data/garden/covid/latest/yougov.meta.yml @@ -17,111 +17,65 @@ definitions: part1: |- Have you had the first or second doses of a Coronavirus (COVID-19) vaccine? question_mapper: |- - <%- if (question == 'activities_improve_health') -%> - {definitions.questions_templates.standard.part1} 'I feel it is important to carry out activities which will improve my health' - <%- elif (question == 'avoided_crowded_areas') -%> - {definitions.questions_templates.i12.part1} 'Avoided crowded areas' {definitions.questions_templates.i12.part2} - <%- elif (question == 'avoided_going_out') -%> - {definitions.questions_templates.i12.part1} 'Avoided going out in general' {definitions.questions_templates.i12.part2} - <%- elif (question == 'avoided_guests_at_home') -%> - {definitions.questions_templates.i12.part1} 'Avoided having guests to your home' {definitions.questions_templates.i12.part2} - <%- elif (question == 'avoided_healthcare_settings') -%> - {definitions.questions_templates.i12.part1} 'Avoided going to hospital or other healthcare settings' {definitions.questions_templates.i12.part2} - <%- elif (question == 'avoided_large_gatherings') -%> - {definitions.questions_templates.i12.part1} 'Avoided large-sized social gatherings (more than 10 people)' {definitions.questions_templates.i12.part2} - <%- elif (question == 'avoided_medium_gatherings') -%> - {definitions.questions_templates.i12.part1} 'Avoided medium-sized social gatherings (between 3 and 10 people)' {definitions.questions_templates.i12.part2} - <%- elif (question == 'avoided_objects_public') -%> - {definitions.questions_templates.i12.part1} 'Avoided touching objects in public (e.g. elevator buttons or doors)' {definitions.questions_templates.i12.part2} - <%- elif (question == 'avoided_people_with_symptoms') -%> - {definitions.questions_templates.i12.part1} 'Avoided contact with people who have symptoms or you think may have been exposed to the coronavirus' {definitions.questions_templates.i12.part2} - <%- elif (question == 'avoided_public_events') -%> - {definitions.questions_templates.i12.part1} 'Avoided attending public events, such as sports matches, festivals, theatres, clubs, or going to religious services' {definitions.questions_templates.i12.part2} - <%- elif (question == 'avoided_public_transport') -%> - {definitions.questions_templates.i12.part1} 'Avoided taking public transport' {definitions.questions_templates.i12.part2} - <%- elif (question == 'avoided_shops') -%> - {definitions.questions_templates.i12.part1} 'Avoided going to shops' {definitions.questions_templates.i12.part2} - <%- elif (question == 'avoided_small_gatherings') -%> - {definitions.questions_templates.i12.part1} 'Avoided small social gatherings (not more than 2 people)' {definitions.questions_templates.i12.part2} - <%- elif (question == 'avoided_working_outside_home') -%> - {definitions.questions_templates.i12.part1} 'Avoided working outside your home' {definitions.questions_templates.i12.part2} - <%- elif (question == 'children_avoided_school') -%> - {definitions.questions_templates.i12.part1} 'Avoided letting your children go to school/university' {definitions.questions_templates.i12.part2} - <%- elif (question == 'cleaned_surfaces_home') -%> - {definitions.questions_templates.i12.part1} 'Cleaned frequently touched surfaces in the home (e.g. doorknobs, toilets, taps)' {definitions.questions_templates.i12.part2} - <%- elif (question == 'covered_mouth_sneeze') -%> - {definitions.questions_templates.i12.part1} 'Covered your nose and mouth when sneezing or coughing' {definitions.questions_templates.i12.part2} + <% set standard = "{definitions.questions_templates.standard.part1}" %> + <% set i12_part1 = "{definitions.questions_templates.i12.part1}" %> + <% set i12_part2 = "{definitions.questions_templates.i12.part2}" %> - <%- elif (question == 'covid_dangerous_to_me') -%> - {definitions.questions_templates.standard.part1} 'Coronavirus (COVID-19) is very dangerous for me' - <%- elif (question == 'covid_vaccine_important_health') -%> - How important do you think getting a COVID-19 vaccine is for your health? - <%- elif (question == 'covid_vaccine_received_one_or_two_doses') -%> - Have you had the first or second doses of a Coronavirus (COVID-19) vaccine? + <% set question_mapper = dict( + activities_improve_health=standard ~ " 'I feel it is important to carry out activities which will improve my health'", + avoided_crowded_areas=i12_part1 ~ " 'Avoided crowded areas' " ~ i12_part2, + avoided_going_out=i12_part1 ~ " 'Avoided going out in general' " ~ i12_part2, + avoided_guests_at_home=i12_part1 ~ " 'Avoided having guests to your home' " ~ i12_part2, + avoided_healthcare_settings=i12_part1 ~ " 'Avoided going to hospital or other healthcare settings' " ~ i12_part2, + avoided_large_gatherings=i12_part1 ~ " 'Avoided large-sized social gatherings (more than 10 people)' " ~ i12_part2, + avoided_medium_gatherings=i12_part1 ~ " 'Avoided medium-sized social gatherings (between 3 and 10 people)' " ~ i12_part2, + avoided_objects_public=i12_part1 ~ " 'Avoided touching objects in public (e.g. elevator buttons or doors)' " ~ i12_part2, + avoided_people_with_symptoms=i12_part1 ~ " 'Avoided contact with people who have symptoms or you think may have been exposed to the coronavirus' " ~ i12_part2, + avoided_public_events=i12_part1 ~ " 'Avoided attending public events, such as sports matches, festivals, theatres, clubs, or going to religious services' " ~ i12_part2, + avoided_public_transport=i12_part1 ~ " 'Avoided taking public transport' " ~ i12_part2, + avoided_shops=i12_part1 ~ " 'Avoided going to shops' " ~ i12_part2, + avoided_small_gatherings=i12_part1 ~ " 'Avoided small social gatherings (not more than 2 people)' " ~ i12_part2, + avoided_working_outside_home=i12_part1 ~ " 'Avoided working outside your home' " ~ i12_part2, + children_avoided_school=i12_part1 ~ " 'Avoided letting your children go to school/university' " ~ i12_part2, + cleaned_surfaces_home=i12_part1 ~ " 'Cleaned frequently touched surfaces in the home (e.g. doorknobs, toilets, taps)' " ~ i12_part2, + covered_mouth_sneeze=i12_part1 ~ " 'Covered your nose and mouth when sneezing or coughing' " ~ i12_part2, + covid_dangerous_to_me=standard ~ " 'Coronavirus (COVID-19) is very dangerous for me'", + covid_vaccine_important_health="How important do you think getting a COVID-19 vaccine is for your health?", + covid_vaccine_received_one_or_two_doses="Have you had the first or second doses of a Coronavirus (COVID-19) vaccine?", + covid_vaccine_will_prevent_transmission=standard ~ " 'A vaccine will completely prevent those who receive it from transmitting COVID19 to others'", + covid_vaccine_will_protect_health=standard ~ " 'A vaccine will completely protect those who receive it from possible health effects of COVID19'", + difficult_to_isolate="If you were advised to do so by a healthcare professional or public health authority, how easy or difficult would it be for you to self-isolate for 7 days?", + eaten_separately=i12_part1 ~ " 'Eaten separately at home, when normally you would eat a meal with others' " ~ i12_part2, + govt_will_provide_effective_covid_vaccine=standard ~ " 'I believe government health authorities in my country will provide me with an effective COVID19 vaccine'", + hand_sanitiser=i12_part1 ~ " 'Used hand sanitiser' " ~ i12_part2, + handwashing_yesterday="Thinking about yesterday… about how many times, would you say you washed your hands with soap or used hand sanitiser?", + household_members_contact="About how many people from your household have you come into physical contact with (within 2 meters / 6 feet)?", + life_greatly_affected=standard ~ " 'My life has been greatly affected by coronavirus (COVID-19)'", + likely_get_covid_future=standard ~ " 'It is likely that I will get coronavirus (COVID-19) in the future'", + mask_at_home=i12_part1 ~ " 'Worn a face mask inside your home' " ~ i12_part2, + mask_at_work=i12_part1 ~ " 'Worn a face mask at your place of work' " ~ i12_part2, + mask_clothing_store=i12_part1 ~ " 'Worn a face mask inside a clothing / footwear shop' " ~ i12_part2, + mask_grocery_store=i12_part1 ~ " 'Worn a face mask inside a grocery store / supermarket' " ~ i12_part2, + mask_not_possible=standard ~ " 'Wearing a mask to protect me against coronavirus (COVID-19) is not possible for me'", + mask_outside_home=i12_part1 ~ " 'Worn a face mask outside your home (e.g. when on public transport, going to a supermarket, going to a main road)' " ~ i12_part2, + mask_protect_me=standard ~ " 'Wearing a mask will protect me against coronavirus (COVID-19)'", + mask_protect_others=standard ~ " 'Wearing a mask will protect others against coronavirus (COVID-19)'", + mask_public_transport=i12_part1 ~ " 'Worn a face mask on public transportation' " ~ i12_part2, + people_contact_outside_household="Not including those people in your household, about how many people have you come into physical contact with (within 2 meters / 6 feet)?", + slept_separate_bedrooms=i12_part1 ~ " 'Slept in separate bedrooms at home, when normally you would share a bedroom' " ~ i12_part2, + times_left_home_yesterday="How many times did you leave your home yesterday? If you are not staying at home, how many times did you leave where you are staying yesterday?", + trust_covid_vaccines="How much do you trust COVID-19 vaccines?", + uncertain_covid_vaccinate_this_week=standard ~ " 'If a Covid-19 vaccine were made available to me this week, I would definitely get it' (neutral)", + unwillingness_covid_vaccinate_this_week=standard ~ " 'If a Covid-19 vaccine were made available to me this week, I would definitely get it' (disagreement)", + washed_hands=i12_part1 ~ " 'Washed hands with soap and water' " ~ i12_part2, + willingness_covid_vaccinate_this_week=standard ~ " 'If a Covid-19 vaccine were made available to me this week, I would definitely get it' (agreement)", + willingness_isolate_if_advised=i12_part1 ~ " 'Washed hands with soap and water' " ~ i12_part2, + willingness_isolate_if_symptoms="Thinking about the next 7 days… would you isolate yourself after feeling unwell or having any of the following new symptoms: a dry cough, fever, loss of sense of smell, loss of sense of taste, shortness of breath or difficulty breathing?", + worried_covid_vaccine_side_effects=standard ~ " 'I am worried about potential side effects of a COVID19 vaccine'" + ) %> - <%- elif (question == 'covid_vaccine_will_prevent_transmission') -%> - {definitions.questions_templates.standard.part1} 'A vaccine will completely prevent those who receive it from transmitting COVID19 to others' - <%- elif (question == 'covid_vaccine_will_protect_health') -%> - {definitions.questions_templates.standard.part1} 'A vaccine will completely protect those who receive it from possible health effects of COVID19' - <%- elif (question == 'difficult_to_isolate') -%> - If you were advised to do so by a healthcare professional or public health authority, how easy or difficult would it be for you be to self-isolate for 7 days? + << question_mapper[question] >> - <%- elif (question == 'eaten_separately') -%> - {definitions.questions_templates.i12.part1} 'Eaten separately at home, when normally you would eat a meal with others' {definitions.questions_templates.i12.part2} - - <%- elif (question == 'govt_will_provide_effective_covid_vaccine') -%> - {definitions.questions_templates.standard.part1} 'I believe government health authorities in my country will provide me with an effective COVID19 vaccine' - <%- elif (question == 'hand_sanitiser') -%> - {definitions.questions_templates.i12.part1} 'Used hand sanitiser' {definitions.questions_templates.i12.part2} - <%- elif (question == 'handwashing_yesterday') -%> - Thinking about yesterday… about how many times, would you say you washed your hands with soap or used hand sanitiser? - <%- elif (question == 'household_members_contact') -%> - About how many people from your household have you come into physical contact with (within 2 meters / 6 feet)? - <%- elif (question == 'life_greatly_affected') -%> - {definitions.questions_templates.standard.part1} 'My life has been greatly affected by coronavirus (COVID-19)' - <%- elif (question == 'likely_get_covid_future') -%> - {definitions.questions_templates.standard.part1} 'It is likely that I will get coronavirus (COVID-19) in the future' - <%- elif (question == 'mask_at_home') -%> - {definitions.questions_templates.i12.part1} 'Worn a face mask inside your home' {definitions.questions_templates.i12.part2} - <%- elif (question == 'mask_at_work') -%> - {definitions.questions_templates.i12.part1} 'Worn a face mask at your place of work' {definitions.questions_templates.i12.part2} - <%- elif (question == 'mask_clothing_store') -%> - {definitions.questions_templates.i12.part1} 'Worn a face mask inside a clothing / footwear shop' {definitions.questions_templates.i12.part2} - <%- elif (question == 'mask_grocery_store') -%> - {definitions.questions_templates.i12.part1} 'Worn a face mask inside a grocery store / supermarket' {definitions.questions_templates.i12.part2} - <%- elif (question == 'mask_not_possible') -%> - {definitions.questions_templates.standard.part1} 'Wearing a mask to protect me against coronavirus (COVID-19) is not possible for me' - <%- elif (question == 'mask_outside_home') -%> - {definitions.questions_templates.i12.part1} 'Worn a face mask outside your home (e.g. when on public transport, going to a supermarket, going to a main road)' {definitions.questions_templates.i12.part2} - <%- elif (question == 'mask_protect_me') -%> - {definitions.questions_templates.standard.part1} 'Wearing a mask will protect me against coronavirus (COVID-19)' - <%- elif (question == 'mask_protect_others') -%> - {definitions.questions_templates.standard.part1} 'Wearing a mask will protect others against coronavirus (COVID-19)' - <%- elif (question == 'mask_public_transport') -%> - {definitions.questions_templates.i12.part1} 'Worn a face mask on public transportation' {definitions.questions_templates.i12.part2} - <%- elif (question == 'people_contact_outside_household') -%> - Not including those people in your household, about how many people have you come into physical contact with (within 2 meters / 6 feet)? - <%- elif (question == 'slept_separate_bedrooms') -%> - {definitions.questions_templates.i12.part1} 'Slept in separate bedrooms at home, when normally you would share a bedroom' {definitions.questions_templates.i12.part2} - <%- elif (question == 'times_left_home_yesterday') -%> - How many times did you leave your home yesterday? If you are not staying at home, how many times did you leave where you are staying yesterday? - <%- elif (question == 'trust_covid_vaccines') -%> - How much do you trust COVID-19 vaccines? - <%- elif (question == 'uncertain_covid_vaccinate_this_week') -%> - {definitions.questions_templates.standard.part1} 'If a Covid-19 vaccine were made available to me this week, I would definitely get it' (neutral) - <%- elif (question == 'unwillingness_covid_vaccinate_this_week') -%> - {definitions.questions_templates.standard.part1} 'If a Covid-19 vaccine were made available to me this week, I would definitely get it' (disagreement) - <%- elif (question == 'washed_hands') -%> - <%- elif (question == 'willingness_covid_vaccinate_this_week') -%> - {definitions.questions_templates.standard.part1} 'If a Covid-19 vaccine were made available to me this week, I would definitely get it' (agreement) - - <%- elif (question == 'willingness_isolate_if_advised') -%> - {definitions.questions_templates.i12.part1} 'Washed hands with soap and water' {definitions.questions_templates.i12.part2} - <%- elif (question == 'willingness_isolate_if_symptoms') -%> - Thinking about the next 7 days… would you isolate yourself after feeling unwell or having any of the following new symptoms: a dry cough, fever, loss of sense of smell, loss of sense of taste, shortness of breath or difficulty breathing? - <%- elif (question == 'worried_covid_vaccine_side_effects') -%> - {definitions.questions_templates.standard.part1} 'I am worried about potential side effects of a COVID19 vaccine' - <%- endif -%> # Learn more about the available fields: # http://docs.owid.io/projects/etl/architecture/metadata/reference/ diff --git a/etl/steps/data/garden/democracy/2024-03-07/ert.py b/etl/steps/data/garden/democracy/2024-03-07/ert.py index 43e439b6002..278d94311cd 100644 --- a/etl/steps/data/garden/democracy/2024-03-07/ert.py +++ b/etl/steps/data/garden/democracy/2024-03-07/ert.py @@ -173,9 +173,9 @@ def correct_country_names(tb: Table) -> Table: tb["country"] = tb["country"].astype("string") tb.loc[(tb["country"] == "Germany") & (tb["year"] <= 1990) & (tb["year"] >= 1949), "country"] = "West Germany" tb.loc[(tb["country"] == "Yemen") & (tb["year"] <= 1990) & (tb["year"] >= 1918), "country"] = "Yemen Arab Republic" - tb.loc[ - (tb["country"] == "Vietnam") & (tb["year"] <= 1975) & (tb["year"] >= 1945), "country" - ] = "Democratic Republic of Vietnam" + tb.loc[(tb["country"] == "Vietnam") & (tb["year"] <= 1975) & (tb["year"] >= 1945), "country"] = ( + "Democratic Republic of Vietnam" + ) tb.loc[(tb["country"] == "Republic of Vietnam") & (tb["year"] < 1945), "country"] = "Vietnam" return tb diff --git a/etl/steps/data/garden/democracy/2024-03-07/vdem/vdem_aggregate.py b/etl/steps/data/garden/democracy/2024-03-07/vdem/vdem_aggregate.py index f881fe0f0da..02683595b20 100644 --- a/etl/steps/data/garden/democracy/2024-03-07/vdem/vdem_aggregate.py +++ b/etl/steps/data/garden/democracy/2024-03-07/vdem/vdem_aggregate.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + from itertools import chain from typing import Dict, Optional, Tuple, cast diff --git a/etl/steps/data/garden/democracy/2024-03-07/vdem/vdem_clean.py b/etl/steps/data/garden/democracy/2024-03-07/vdem/vdem_clean.py index 04ad94d25f8..d59f84fb52f 100644 --- a/etl/steps/data/garden/democracy/2024-03-07/vdem/vdem_clean.py +++ b/etl/steps/data/garden/democracy/2024-03-07/vdem/vdem_clean.py @@ -2,6 +2,7 @@ ref: https://github.com/owid/notebooks/blob/main/BastianHerre/democracy/scripts/vdem_row_do """ + from typing import Union, cast import numpy as np @@ -79,9 +80,9 @@ def initial_cleaning(tb: Table) -> Table: # Assign historical names tb["country"] = tb["country"].astype("string") tb.loc[(tb["country"] == "Yemen") & (tb["year"] <= 1990) & (tb["year"] >= 1918), "country"] = "Yemen Arab Republic" - tb.loc[ - (tb["country"] == "Vietnam") & (tb["year"] <= 1975) & (tb["year"] >= 1945), "country" - ] = "Democratic Republic of Vietnam" + tb.loc[(tb["country"] == "Vietnam") & (tb["year"] <= 1975) & (tb["year"] >= 1945), "country"] = ( + "Democratic Republic of Vietnam" + ) tb.loc[(tb["country"] == "Germany") & (tb["year"] <= 1990) & (tb["year"] >= 1949), "country"] = "West Germany" tb["country"] = tb["country"].astype("category") diff --git a/etl/steps/data/garden/demography/2022-12-08/population/gapminder_sg.py b/etl/steps/data/garden/demography/2022-12-08/population/gapminder_sg.py index cef7a587ce8..ee74956b3e8 100644 --- a/etl/steps/data/garden/demography/2022-12-08/population/gapminder_sg.py +++ b/etl/steps/data/garden/demography/2022-12-08/population/gapminder_sg.py @@ -9,6 +9,7 @@ obtained that data. """ + from owid.catalog import Dataset from etl.paths import DATA_DIR diff --git a/etl/steps/data/garden/demography/2023-02-03/life_expectancy.py b/etl/steps/data/garden/demography/2023-02-03/life_expectancy.py index 7164918d0bf..fdf118f7fd8 100644 --- a/etl/steps/data/garden/demography/2023-02-03/life_expectancy.py +++ b/etl/steps/data/garden/demography/2023-02-03/life_expectancy.py @@ -11,6 +11,7 @@ - UN WPP for data since 1950. - HMD for data prior to 1950. """ + from typing import List import pandas as pd diff --git a/etl/steps/data/garden/demography/2023-03-31/population/gapminder_sg.py b/etl/steps/data/garden/demography/2023-03-31/population/gapminder_sg.py index b0897d09ae5..35529498fd9 100644 --- a/etl/steps/data/garden/demography/2023-03-31/population/gapminder_sg.py +++ b/etl/steps/data/garden/demography/2023-03-31/population/gapminder_sg.py @@ -9,6 +9,7 @@ obtained that data. """ + from owid.catalog import Dataset, License, Origin, Table from etl.paths import DATA_DIR diff --git a/etl/steps/data/garden/demography/2024-01-25/utils/un.py b/etl/steps/data/garden/demography/2024-01-25/utils/un.py index d01aa993422..2f28c168832 100644 --- a/etl/steps/data/garden/demography/2024-01-25/utils/un.py +++ b/etl/steps/data/garden/demography/2024-01-25/utils/un.py @@ -5,7 +5,6 @@ We only use projections with medium-fertility variant. """ - from owid.catalog import Table # Year boundaries diff --git a/etl/steps/data/garden/demography/2024-11-26/multiple_births.countries.json b/etl/steps/data/garden/demography/2024-11-26/multiple_births.countries.json new file mode 100644 index 00000000000..98cf9570896 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-11-26/multiple_births.countries.json @@ -0,0 +1,27 @@ +{ + "Australia": "Australia", + "Austria": "Austria", + "Canada": "Canada", + "Chile": "Chile", + "Czech Republic": "Czechia", + "Denmark": "Denmark", + "Finland": "Finland", + "France": "France", + "Germany": "Germany", + "Greece": "Greece", + "Iceland": "Iceland", + "Italy": "Italy", + "Japan": "Japan", + "Lithuania": "Lithuania", + "Netherlands": "Netherlands", + "New Zealand": "New Zealand", + "Norway": "Norway", + "Republic of Korea": "South Korea", + "Spain": "Spain", + "Sweden": "Sweden", + "Switzerland": "Switzerland", + "United States": "United States", + "Uruguay": "Uruguay", + "UK-England and Wales": "England and Wales", + "UK-Scotland": "Scotland" +} diff --git a/etl/steps/data/garden/demography/2024-11-26/multiple_births.excluded_countries.json b/etl/steps/data/garden/demography/2024-11-26/multiple_births.excluded_countries.json new file mode 100644 index 00000000000..0d4f101c7a3 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-11-26/multiple_births.excluded_countries.json @@ -0,0 +1,2 @@ +[ +] diff --git a/etl/steps/data/garden/demography/2024-11-26/multiple_births.meta.yml b/etl/steps/data/garden/demography/2024-11-26/multiple_births.meta.yml new file mode 100644 index 00000000000..207df2e06ee --- /dev/null +++ b/etl/steps/data/garden/demography/2024-11-26/multiple_births.meta.yml @@ -0,0 +1,110 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Fertility Rate + + display: + entityAnnotationsMap: |- + Chile: Stillbirths excluded + South Korea: Stillbirths excluded + Czech Republic: Stillbirths included + Denmark: Stillbirths included + England and Wales: Stillbirths included + France: Stillbirths included + Greece: Stillbirths included + Italy: Stillbirths included + Lithuania: Stillbirths included + Netherlands: Stillbirths included + Norway: Stillbirths included + Spain: Stillbirths included + Sweden: Stillbirths included + Switzerland: Stillbirths included + Australia: Stillbirths included in some years + New Zealand: Stillbirths included in some years + Scotland: Stillbirths included in some years + United States: Stillbirths included in some years + Uruguay: Stillbirths included in some years + Austria: Stillbirths typically included + Canada: Stillbirths typically included + Finland: Stillbirths typically included + Germany: Stillbirths typically included + Japan: Stillbirths typically included + + description_processing: |- + Data sometimes includes stillbirths, therefore comparability across countries should be done with care. + + **Countries including stillbirths**: Czech Republic, Denmark, England and Wales, France, Greece, Italy, Lithuania, Netherlands, Norway, Spain, Sweden, Switzerland + + **Countries mostly including stillbirths**: Austria (unknown for 1920, 1921, 1928, 1929, 1931, 1934), Canada (unknown for 1921-1925, 1927-1990), Finland (unknown for 1906-1936, 1941-1999), Germany (unknown for 1906-1936), Japan (unknown for 1923-1936) + + **Countries excluding stillbirths**: Chile, South Korea + + **Countries with mixed practices**: Australia, New Zealand (excluded for 1856-1915), United States, Scotland (excluded for 1856-1938), Uruguay + + For more details about the data for a specific country, please refer to the original source. + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + update_period_days: 365 + +tables: + multiple_births: + variables: + singletons: + title: Singleton deliveries + unit: "deliveries" + description_short: |- + Number of single deliveries. + twin_deliveries: + title: Twin deliveries + unit: "deliveries" + description_short: |- + Number of twin deliveries. + multiple_deliveries: + title: Multiple deliveries + unit: "deliveries" + description_short: |- + Total number of multiple deliveries (i.e. the sum of twin, triplet, and quadruplet+ deliveries). + total_deliveries: + title: Total deliveries + unit: "deliveries" + description_short: |- + Total number of deliveries (i.e., single and multiple deliveries combined) + singleton_rate: + title: Singleton rate + unit: "twin deliveries per total deliveries" + description_short: |- + The rate of deliveries that are single deliveries, per 1,000 deliveries. + twinning_rate: + title: Twinning delivery rate + unit: "twin deliveries per 1,000 deliveries" + description_short: |- + The rate of twin deliveries, per 1,000 deliveries. + multiple_rate: + title: Multiple delivery rate + unit: "multiple deliveries per 1,000 deliveries" + description_short: |- + The rate of deliveries that are multiple deliveries, per 1,000 deliveries. + children_delivery_ratio: + title: "Children per delivery" + unit: "children per 1,000 deliveries" + description_short: |- + The average number of children born per delivery. This is estimated by dividing the number of children born by the total number deliveries, per 1,000 people. + children_multiple_delivery_ratio: + title: "Children per multiple delivery" + unit: "children per 1,000 multiple deliveries" + description_short: |- + Number of children born in multiple deliveries divided by the total number of multiple deliveries, per 1,000 people. Gives an indication of the average number of children born in multiple deliveries. + multiple_to_singleton_ratio: + title: "Multiple children to singleton ratio" + unit: "multiple deliveries per 1,000 single delivery" + description_short: |- + Number of multiple deliveries per 1,000 single deliveries. + multiple_children: + title: "Multiple children" + unit: "children" + description_short: |- + Number of children born from multiple deliveries (twin babies, triplet babies, etc.) diff --git a/etl/steps/data/garden/demography/2024-11-26/multiple_births.py b/etl/steps/data/garden/demography/2024-11-26/multiple_births.py new file mode 100644 index 00000000000..eafd3129e47 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-11-26/multiple_births.py @@ -0,0 +1,225 @@ +"""Load a meadow dataset and create a garden dataset.""" + +import pandas as pd + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +FLAGS_EXPECTED = [ + { + "countries": [ + "Chile", + "South Korea", + ], + "flags": { + 0, + }, + }, + { + "countries": [ + "Czech Republic", + "Denmark", + "France", + "Greece", + "Italy", + "Lithuania", + "Netherlands", + "Norway", + "Spain", + "Switzerland", + "England/Wales", + ], + "flags": { + 1, + }, + }, + { + "countries": [ + "Australia", + "United States", + "Uruguay", + ], + "flags": { + 2, + }, + }, + { + "countries": [ + "Austria", + "Canada", + "Finland", + "Germany", + "Japan", + ], + "flags": { + 1, + 99, + }, + }, + { + "countries": { + "Iceland", + "New Zealand", + "Sweden", + "Scotland", + }, + "flags": { + 0, + 1, + }, + }, +] + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("multiple_births") + + # Read table from meadow dataset. + tb = ds_meadow.read("multiple_births") + + # Harmonize country names + tb = geo.harmonize_countries( + df=tb, + countries_file=paths.country_mapping_path, + excluded_countries_file=paths.excluded_countries_path, + ) + + # Sanity check + check_stillbirths(tb) + + # Adapt flags + tb = adapt_stillbirths_flags(tb) + + # Estimate singleton_rate + tb["singleton_rate"] = (1_000 * tb["singletons"] / tb["total_deliveries"]).round(2) + + # Estimate ratios + tb["children_delivery_ratio"] = ( + 1_000 * (tb["multiple_children"] + tb["singletons"]) / tb["total_deliveries"] + ).round(3) + tb["children_multiple_delivery_ratio"] = (1_000 * tb["multiple_children"] / tb["multiple_deliveries"]).round(3) + tb["multiple_to_singleton_ratio"] = (1_000 * tb["multiple_deliveries"] / tb["singletons"]).round(3) + + # Remove outliers + flag = (tb["country"] == "England and Wales") & (tb["year"] == 1938) + assert ( + tb.loc[flag, "children_multiple_delivery_ratio"] >= 4000 + ).all(), "Unexpected outlier for England and Wales in 1938" + tb.loc[flag, ["multiple_children", "children_multiple_delivery_ratio", "children_delivery_ratio"]] = pd.NA + flag = (tb["country"] == "England and Wales") & (tb["year"] == 1939) + assert ( + tb.loc[flag, "children_multiple_delivery_ratio"] <= 1500 + ).all(), "Unexpected outlier for England and Wales in 1938" + tb.loc[flag, ["multiple_children", "children_multiple_delivery_ratio", "children_delivery_ratio"]] = pd.NA + + # Keep relevant columns + tb = tb[ + [ + # Index + "country", + "year", + # Absolute numbers + "singletons", + "twin_deliveries", + "multiple_deliveries", + "total_deliveries", + # Relative numbers + "singleton_rate", + "twinning_rate", + "multiple_rate", + # Ratios + "children_delivery_ratio", + "children_multiple_delivery_ratio", + "multiple_to_singleton_ratio", + # Births + "multiple_children", + ] + ] + + tb = tb.format(["country", "year"]) + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata + ) + + # Save changes in the new garden dataset. + ds_garden.save() + + +def check_stillbirths(tb): + """Datapoints (country-year) are given using different methodologies. + + This is communciated in the 'stillbirths' column, which can vary from country to country (and year to year): + + 0: Stillbirths not included + 1: Stillbirths included + 2: Mixed (stillbirths included in some cases only) + 99: Unsure + + Reference: https://www.twinbirths.org/en/data-metadata/, Table 1 + """ + # Check that the stillbirths flags are as expected. + for expected in FLAGS_EXPECTED: + countries_expected = expected["countries"] + flags_expected = expected["flags"] + + flags_actual = set(tb.loc[tb["country"].isin(countries_expected), "stillbirths"].unique()) + + assert ( + flags_actual == flags_expected + ), f"Expected flags {flags_expected} for countries {countries_expected} are not as expected! Found: {flags_actual}" + + # Check Overlaps + ## There are overlaps in New Zealand and Sweden + x = tb.groupby(["country", "year"], as_index=False).stillbirths.nunique() + countries_overlap_expected = {"New Zealand", "Sweden"} + countries_overlap_actually = set(x.loc[x["stillbirths"] != 1, "country"].unique()) + assert ( + countries_overlap_actually == countries_overlap_expected + ), f"Expected countries with overlaps {countries_overlap_expected} are not as expected! Found: {countries_overlap_actually}" + + +def adapt_stillbirths_flags(tb): + # Iceland: Remove even there is no replacement. Keep only 1. + country = "Iceland" + flag = (tb["country"] == country) & (tb["stillbirths"] == 0) + assert len(tb.loc[flag]) == 5, f"Unexpected number of values for {country}" + tb = tb.loc[~flag] + + # If there is 1 and 0, keep 1. + flag = tb.sort_values("stillbirths").duplicated(subset=["country", "year"], keep="last") + assert set(tb.loc[flag, "stillbirths"].unique()) == { + 0 + }, "Removed rows because of duplicate country-year values should only be stillbirths=0!" + tb = tb.loc[~flag] + + # Sweden: Remove, ensure there is actually redundancy. Keep 1. + assert set(tb.loc[tb["country"] == "Sweden", "stillbirths"].unique()) == {1}, "Unexpected stillbirths=0 for Sweden!" + + return tb + + +def get_summary_methodology_sb(tb): + tbx = tb.groupby("country", as_index=False)["stillbirths"].agg(["nunique", "unique"]) + + # Only one method + tbx1 = tbx.loc[tbx["nunique"] == 1] + tbx1["unique"] = tbx1["unique"].apply(lambda x: x[0]) + tbx1 = tbx1[["country", "unique"]].sort_values("unique") + + # Multiple methods + tbx2 = tbx.loc[tbx["nunique"] > 1] + countries_mult = set(tbx2["country"].unique()) + tb[tb["country"].isin(countries_mult)].groupby(["country", "stillbirths"]).agg({"year": ("min", "max")}) diff --git a/etl/steps/data/garden/demography/2024-12-02/survivor_percentiles.meta.yml b/etl/steps/data/garden/demography/2024-12-02/survivor_percentiles.meta.yml new file mode 100644 index 00000000000..07e19bde3f9 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-02/survivor_percentiles.meta.yml @@ -0,0 +1,44 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Life Expectancy + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/en/latest/architecture/metadata/reference/dataset/ +dataset: + title: Survivorship percentiles (HMD; Alvarez and Vaupel 2023) + update_period_days: 365 + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/en/latest/architecture/metadata/reference/tables/ +tables: + survivor_percentiles: + variables: + age: + title: Survivorship age + unit: years + processing_level: major + description_short: |- + <%- if percentile == 1 -%> + The age until which the 1st percentile (99% of the population) of the population would survive until, if they experienced the same age-specific death rates throughout their whole lives as the age-specific death rates seen in that particular year. + <%- else -%> + The age until which the << percentile>>th percentile (<< 100 - percentile|int>>% of the population) of the population would survive until, if they experienced the same age-specific death rates throughout their whole lives as the age-specific death rates seen in that particular year. + <%- endif -%> + + description_processing: |- + This was calculated with the method published in Alvarez and Vaupel (2023), with code provided by the authors: + + Jesús-Adrián Alvarez, James W. Vaupel; Mortality as a Function of Survival. Demography 1 February 2023; 60 (1): 327–342. doi: https://doi.org/10.1215/00703370-10429097 + + These estimates were regenerated for data from more recent years in the Human Mortality Database. + + Original R code from: https://github.com/jssalvrz/s-ages + description_key: + - This is calculated with the period life tables indicators. + display: + numDecimalPlaces: 1 + presentation: + attribution: |- + Alvarez & Vaupel (2023); Human Mortality Database (2024) diff --git a/etl/steps/data/garden/demography/2024-12-02/survivor_percentiles.py b/etl/steps/data/garden/demography/2024-12-02/survivor_percentiles.py new file mode 100644 index 00000000000..1f2b1ef59cc --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-02/survivor_percentiles.py @@ -0,0 +1,137 @@ +"""Load a meadow dataset and create a garden dataset. + +Methods used here are taken from https://github.com/jssalvrz/s-ages. Authors of Citation: Alvarez, J.-A., & Vaupel, J. W. (2023). Mortality as a Function of Survival. Demography, 60(1), 327–342. https://doi.org/10.1215/00703370-10429097 + + +Dr. Saloni Dattani translated the R scripts into Python: + - Original: https://github.com/jssalvrz/s-ages + - Translated: https://github.com/saloni-nd/misc/tree/main/survivorship-ages + +Lucas Rodes-Guirao adapted the python code for ETL. +""" + +import numpy as np +import pandas as pd +from owid.catalog import Table +from scipy.integrate import cumulative_trapezoid as cumtrapz +from scipy.interpolate import InterpolatedUnivariateSpline + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + paths.log.info("load data.") + # Load meadow dataset. + ds_meadow = paths.load_dataset("hmd") + + # Read table from meadow dataset. + tb_deaths = ds_meadow.read("deaths") + tb_exposure = ds_meadow.read("exposures") + + # + # Process data. + # + # Combine tables, drop NaNs + tb = tb_deaths.merge(tb_exposure, on=["country", "year", "sex", "age"], how="outer") + tb = tb.dropna(subset=["deaths", "exposure"], how="any") + + # Keep format="1x1", and sex="both" + paths.log.info("keep period & 1-year data.") + tb = tb.loc[tb["age"].str.match(r"^(\d{1,3}|d{3}\+)$") & (tb["type"] == "period")] + + # Drop unused columns + tb = tb.drop(columns=["type"]) + + # 110+ -> 110 + paths.log.info("replace 110+ -> 100, set Dtypes.") + tb["age"] = tb["age"].replace({"110+": "110"}).astype(int) + + # Sort + tb = tb.sort_values(["year", "age"]) + + # Actual calculation + paths.log.info("calculate surviorship ages (can take some minutes)...") + columns_grouping = ["country", "sex", "year"] + tb = tb.groupby(columns_grouping).apply(lambda group: obtain_survivorship_ages(group)).reset_index() # type: ignore + + # Unpivot + paths.log.info("reshape table") + tb = tb.melt( + id_vars=["country", "sex", "year"], + value_vars=["s1", "s10", "s20", "s30", "s40", "s50", "s60", "s70", "s80", "s90", "s99"], + var_name="percentile", + value_name="age", + ) + tb = tb.dropna(subset=["percentile"]) + tb["percentile"] = tb["percentile"].str.replace("s", "").astype(int) + tb["percentile"] = 100 - tb["percentile"] + + # Propagate metadata + tb["age"].metadata.origins = tb_exposure["exposure"].m.origins.copy() + + # Set index + paths.log.info("format") + tb = tb.format(["country", "year", "sex", "percentile"], short_name="survivor_percentiles") + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata + ) + + # Save changes in the new garden dataset. + ds_garden.save() + + +def obtain_survivorship_ages(tb_group: Table, start_age: int = 0, end_age: int = 110) -> pd.DataFrame: + """Get survivorship ages given a life and deaths table. + + Output dataframe has a column for each percentile of survivorship age. + + tb_group is expected to be a subset of the compelte table. It should only concern a particular (country, year, sex) triple. + """ + # Step 1: Apply splines, get Mx for each (country, year, sex, age) + ## Define splines + ### We could use CubicSpline (k=3 order), but it provides slightly different results hence, for precaution, we sticked to InterpolatedUnivariateSpline. + ### This is equivalent to R function interpSpline + spline_deaths = InterpolatedUnivariateSpline(tb_group["age"], tb_group["deaths"], k=3) + spline_exposures = InterpolatedUnivariateSpline(tb_group["age"], tb_group["exposure"], k=3) + + ## Define age range (with step 0.01) + age_range = np.arange(start_age, end_age, 0.01) + + # Run splines over age range + deaths_spline = np.abs(spline_deaths(age_range)) + exposure_spline = np.abs(spline_exposures(age_range)) + exposure_spline[exposure_spline == 0] = np.nan + survival_age_spline = np.abs(deaths_spline / exposure_spline) + + # Step 2: Calculate survival, density, hazard, and cumulative hazards + ## Estimate parameters + Hx = cumtrapz(y=survival_age_spline, x=age_range, initial=0) # Hazard CDF + Sx = np.exp(-Hx) # Survivor function + + # Step 3: Calculate survivorship ages from parameters + out = {} + out["s0"] = max(age_range) + ## I'm using a for loop to simplify the logic here + for i in range(1, 101): + try: + sx_rounded = np.ceil((100 * Sx).round(3)) + value = age_range[sx_rounded == i][0] + out[f"s{i}"] = value + except IndexError: + out[f"s{i}"] = np.nan + + # Create output dataframe + df = pd.DataFrame(out, index=[0]) + + return df diff --git a/etl/steps/data/garden/demography/2024-12-03/birth_rate.meta.yml b/etl/steps/data/garden/demography/2024-12-03/birth_rate.meta.yml new file mode 100644 index 00000000000..0ee61f6edb7 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-03/birth_rate.meta.yml @@ -0,0 +1,45 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + title_public: Birth rate + topic_tags: + - Fertility Rate + display: + name: |- + Birth rate + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + title: Birth Rate (HMD; UN WPP) + update_period_days: 365 + +tables: + birth_rate: + variables: + birth_rate: + title: Birth rate + unit: births per 1,000 people + description_short: |- + The total number of births per 1,000 people in a given year. + description_processing: |- + The birth data is constructed by combining data from multiple sources: + + - Before 1949: Historical estimates by Human Mortality Database (2024). + + - 1950-2023: Population records by the UN World Population Prospects (2024 revision). + + - 2024-2100: Projections based on Medium variant by the UN World Population Prospects (2024 revision). + + birth_rate_hist: + title: Birth rate (historical) + unit: births per 1,000 people + description_short: |- + The total number of births per 1,000 people in a given year. + description_processing: |- + The birth data is constructed by combining data from multiple sources: + + - Before 1949: Historical estimates by Human Mortality Database (2024). + + - 1950-2023: Population records by the UN World Population Prospects (2024 revision). diff --git a/etl/steps/data/garden/demography/2024-12-03/birth_rate.py b/etl/steps/data/garden/demography/2024-12-03/birth_rate.py new file mode 100644 index 00000000000..02508c2497b --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-03/birth_rate.py @@ -0,0 +1,62 @@ +"""Load a meadow dataset and create a garden dataset.""" + +import owid.catalog.processing as pr +import pandas as pd + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +YEAR_WPP_PROJ_START = 2024 +YEAR_WPP_START = 1950 + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_hmd = paths.load_dataset("hmd") + ds_un = paths.load_dataset("un_wpp") + + # Read table from meadow dataset. + tb_hmd = ds_hmd.read("births") + tb_un = ds_un.read("births") + + # + # Process data. + # + # UN + tb_un = tb_un.loc[ + (tb_un["age"] == "all") & (tb_un["variant"].isin(["medium", "estimates"])), + ["country", "year", "birth_rate"], + ] + # HMD + tb_hmd = tb_hmd.loc[ + (tb_hmd["year"] < YEAR_WPP_START) & (tb_hmd["sex"] == "total"), ["country", "year", "birth_rate"] + ] + + # Combine + tb = pr.concat([tb_hmd, tb_un], ignore_index=True, short_name="birth_rate") + tb = tb.dropna(subset=["birth_rate"]) + + # Add historical variant + tb["birth_rate_hist"] = tb["birth_rate"].copy() + tb.loc[tb["year"] > YEAR_WPP_PROJ_START, "birth_rate_hist"] = pd.NA + + # Format + tb = tb.format(["country", "year"]) + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, + tables=[tb], + check_variables_metadata=True, + ) + + # Save changes in the new garden dataset. + ds_garden.save() diff --git a/etl/steps/data/garden/demography/2024-12-03/broken_limits_le.meta.yml b/etl/steps/data/garden/demography/2024-12-03/broken_limits_le.meta.yml new file mode 100644 index 00000000000..b8511924d1d --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-03/broken_limits_le.meta.yml @@ -0,0 +1,51 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Life Expectancy + + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/dataset/ +dataset: + title: "Life Expectancy: Broken limits" + update_period_days: 365 + + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/tables/ +tables: + broken_limits_le: + variables: + life_expectancy: + title: &le_name Maximum life expectancy + unit: years + description_short: |- + <%- if (sex == 'female') -%> + Maximum life expectancy recorded in a given year (among females). + <%- elif (sex == 'male') -%> + Maximum life expectancy recorded in a given year (among males). + <%- elif (sex == 'all') -%> + Maximum life expectancy recorded in a given year. + <%- endif -%> + description_key: + - Period life expectancy is a metric that summarizes death rates across all age groups in one particular year. For a given year, it represents the average lifespan for a hypothetical group of people, if they experienced the same age-specific death rates throughout their lives as the age-specific death rates seen in that particular year. + - Records are only shown for countries in the Human Mortality Database. Prior to 1950, we use HMD (2023) data. From 1950 onwards, we use UN WPP (2022) data. + display: + name: *le_name + presentation: + title_public: *le_name + title_variant: "" + attribution_short: HMD; UN WPP + topic_tags: + - Life Expectancy + grapher_config: + hasMapTab: true + + country_with_max_le: + title: Country with yearly maximum life expectancy + unit: "" + description_short: |- + Name of the country with the yearly maximum life expectancy registered<%- if (sex == 'female') %> among females<% elif (sex == 'male') %> among males<% endif -%>. + description_processing: This indicator is meant to be used as an auxiliary indicator. diff --git a/etl/steps/data/garden/demography/2024-12-03/broken_limits_le.py b/etl/steps/data/garden/demography/2024-12-03/broken_limits_le.py new file mode 100644 index 00000000000..2e64cba9275 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-03/broken_limits_le.py @@ -0,0 +1,76 @@ +"""Load a meadow dataset and create a garden dataset. + +We only consider data from countries that are present in HMD. And, additionally, we only consider entries for these countries since the year they first appear in the HMD dataset (even if for that period we use UN WPP data, i.e. post-1950) +""" + +from owid.catalog import Table + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) +# Year to start tracking. Note that in the first years, few countries have data. Hence, we start in a later year, where more countries have data. +YEAR_FIRST = 1840 + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("life_tables") + ds_hmd = paths.load_dataset("hmd") + + # Read table from meadow dataset. + tb = ds_meadow.read("life_tables", reset_index=False) + tb_hmd = ds_hmd.read("life_tables") + + # + # Process data. + # + # Filter relevant dimensions + tb = tb.loc[(slice(None), slice(None), slice(None), "0", "period"), ["life_expectancy"]].reset_index() + + # Keep relevant columns and rows + tb = tb.drop(columns=["type", "age"]).dropna() + + # Rename column + tb = tb.rename(columns={"location": "country"}) + + # Get country-sex and first year of LE reported in HMD + tb_hmd = get_first_year_of_country_in_hmd(tb_hmd) + + # Only preserve countries coming from HDM + tb = tb.merge(tb_hmd, on=["country", "sex"], suffixes=("", "_min")) + tb = tb[tb["year"] >= tb["year_min"]].drop(columns=["year_min"]) + + # Get max for each year + tb = tb.loc[tb.groupby(["year", "sex"], observed=True)["life_expectancy"].idxmax()] + + # Organise columns + tb["country_with_max_le"] = tb["country"] + tb["country"] = tb["country"] + " " + tb["year"].astype("string") + + # First year + tb = tb[tb["year"] >= YEAR_FIRST] + + # Set index + tb = tb.format(["country", "year", "sex"], short_name="broken_limits_le") + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata + ) + + # Save changes in the new garden dataset. + ds_garden.save() + + +def get_first_year_of_country_in_hmd(tb_hmd: Table) -> Table: + tb_hmd = tb_hmd.loc[(tb_hmd["type"] == "period") & (tb_hmd["age"] == "0")] + tb_hmd = tb_hmd.loc[:, ["country", "year", "sex", "life_expectancy"]].dropna() + tb_hmd = tb_hmd.groupby(["country", "sex"], observed=True, as_index=False)["year"].min() + return tb_hmd diff --git a/etl/steps/data/garden/demography/2024-12-03/fertility_rate.meta.yml b/etl/steps/data/garden/demography/2024-12-03/fertility_rate.meta.yml new file mode 100644 index 00000000000..3390773b651 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-03/fertility_rate.meta.yml @@ -0,0 +1,55 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Fertility Rate + attribution: UN WPP (2024); HFD (2024) + processing_level: major + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + title: Fertility Rate (UN WPP; HFD) + update_period_days: 365 + +tables: + fertility_rate: + variables: + fertility_rate: + title: Fertility rate (period) + description_short: |- + The average number of live births a hypothetical cohort of women would have at the end of their reproductive period if they were subject during their whole lives to the fertility rates of a given period and if they were not subject to mortality. + description_key: + - Assumes current age-specific fertility rates remain constant throughout a woman's lifetime. + - Does not account for potential changes in social, economic, or health conditions that could affect fertility rates. + unit: live births per woman + description_processing: |- + The fertility data is constructed by combining data from multiple sources: + + - Before 1949: Historical estimates by Human Fertility Database (2024). + + - 1950-2023: Population records by the UN World Population Prospects (2024 revision). + + - 2024-2100: Projections based on Medium variant by the UN World Population Prospects (2024 revision). + presentation: + title_public: Fertility rate + title_variant: period tables + + fertility_rate_hist: + title: Fertility rate (period), historical + description_short: |- + The average number of live births a hypothetical cohort of women would have at the end of their reproductive period if they were subject during their whole lives to the fertility rates of a given period and if they were not subject to mortality. + description_key: + - Assumes current age-specific fertility rates remain constant throughout a woman's lifetime. + - Does not account for potential changes in social, economic, or health conditions that could affect fertility rates. + unit: live births per woman + description_processing: |- + The fertility data is constructed by combining data from multiple sources: + + - Before 1949: Historical estimates by Human Fertility Database (2024). + + - 1950-2023: Population records by the UN World Population Prospects (2024 revision). + presentation: + title_public: Fertility rate + title_variant: period tables diff --git a/etl/steps/data/garden/demography/2024-12-03/fertility_rate.py b/etl/steps/data/garden/demography/2024-12-03/fertility_rate.py new file mode 100644 index 00000000000..24f4de08f93 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-03/fertility_rate.py @@ -0,0 +1,62 @@ +"""Load a meadow dataset and create a garden dataset.""" + +import pandas as pd +from owid.catalog import processing as pr + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +# Year constants +YEAR_WPP_START = 1950 +YEAR_WPP_PROJ_START = 2023 + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_hfd = paths.load_dataset("hfd") + ds_un = paths.load_dataset("un_wpp") + + # Read table from meadow dataset. + tb_hfd = ds_hfd.read("period") + tb_un = ds_un.read("fertility_rate") + + # UN: estimates + medium, + tb_un = tb_un.loc[ + (tb_un["sex"] == "all") & (tb_un["variant"].isin(["medium", "estimates"]) & (tb_un["age"] == "all")), + ["country", "year", "fertility_rate"], + ] + + # HFD: tfr, birth_order=total, + tb_hfd = tb_hfd.loc[ + ((tb_hfd["birth_order"] == "total") & (tb_hfd["year"] < YEAR_WPP_START)), ["country", "year", "tfr"] + ].rename(columns={"tfr": "fertility_rate"}) + + # + # Process data. + # + tb = pr.concat([tb_hfd, tb_un], ignore_index=True, short_name="fertility_rate") + + # Add historical variant + tb["fertility_rate_hist"] = tb["fertility_rate"].copy() + tb.loc[tb["year"] > YEAR_WPP_PROJ_START, "fertility_rate_hist"] = pd.NA + + # Format + tb = tb.format(["country", "year"]) + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, + tables=[tb], + check_variables_metadata=True, + ) + + # Save changes in the new garden dataset. + ds_garden.save() diff --git a/etl/steps/data/garden/demography/2024-12-03/gini_le.meta.yml b/etl/steps/data/garden/demography/2024-12-03/gini_le.meta.yml new file mode 100644 index 00000000000..d0b96883e79 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-03/gini_le.meta.yml @@ -0,0 +1,35 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Life Expectancy + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/dataset/ +dataset: + title: Gini coefficient of lifespan inequality (HMD, UN WPP, Aburto et al.; 2023) + update_period_days: 365 + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/tables/ +tables: + gini_le: + variables: + life_expectancy_gini: + title: Gini coefficient of lifespan inequality + unit: "" + processing_level: major + description_short: |- + The level of inequality in lifespans, measured between 0 and 1. + description_key: + - |- + {tables.gini_le.variables.life_expectancy_gini.description_short} + - A higher coefficient indicates greater inequality in ages of death, while a lower coefficient indicates more uniform ages of death. + description_processing: |- + This was calculated using the algorithm and scripts from Aburto et al. (2020). We regenerated the Gini coefficient, rather than the inverse-log Gini coefficient. + + Citation: Aburto, J. M., Villavicencio, F., Basellini, U., Kjærgaard, S., & Vaupel, J. W. (2020). Dynamics of life expectancy and life span equality. Proceedings of the National Academy of Sciences, 117(10), 5250–5259. https://doi.org/10.1073/pnas.1915884117 Code available on Zenodo: https://zenodo.org/record/3571095 + presentation: + attribution: |- + Human Mortality Database (2024); Aburto et al. (2023) diff --git a/etl/steps/data/garden/demography/2024-12-03/gini_le.py b/etl/steps/data/garden/demography/2024-12-03/gini_le.py new file mode 100644 index 00000000000..4035c4718b1 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-03/gini_le.py @@ -0,0 +1,148 @@ +"""Estimate the gini index on life expectency""" + +from typing import Any, cast + +import numpy as np +from numpy.typing import NDArray +from owid.catalog import Table, Variable + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + paths.log.info("gini_le: load data") + # Load meadow dataset. + ds_meadow = paths.load_dataset("life_tables") + + # Read table from meadow dataset. + tb = ds_meadow.read("life_tables") + + # + # Process data. + # + # Keep relevant dimensions + paths.log.info("gini_le: keep relevant dimensions (type='period', sex in ['male', 'female'])") + tb = tb[(tb["type"] == "period") & (tb["sex"].isin(["male", "female"]))] + paths.log.info("gini_le: set year dtype to int") + + # Get origins + origins = tb["life_expectancy"].m.origins + + # Get rate for central_death_rate, as it is given per 1,000 people. + paths.log.info("gini_le: get rate for central_death_rate, as it is given per 1,000 people.") + tb["central_death_rate"] = tb["central_death_rate"] / 1000 + + # 110+ -> 110 + paths.log.info("gini_le: replace 110+ -> 100, 110+ -> 110, set Dtypes.") + tb["age"] = ( + tb["age"] + .replace( + { + "110+": "110", + "100+": "100", + } + ) + .astype("Int64") + ) + + # Sort rows + paths.log.info("gini_le: sort rows (needed for correct estimation)") + tb = tb.sort_values(["country", "year", "sex", "age"]) + + # Estimates (this can take a half minute or so, depending on computation power) + tb = tb.groupby(["country", "year", "sex"], as_index=False, observed=False).apply(gini_from_mx) + tb.life_expectancy_gini.m.origins = origins + + # Rename columns + paths.log.info("gini_le: rename columns") + tb = tb.rename(columns={"central_death_rate": "life_expectancy_gini"}) + + # Set index + paths.log.info("gini_le: set index") + tb = tb.format(["country", "year", "sex"], short_name=paths.short_name) + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata + ) + + # Save changes in the new garden dataset. + ds_garden.save() + + +def AKm02a0(m0: float, is_male: bool = True) -> NDArray[Any]: + """Get estimates. + + Calculate the average number of years lived in the first year of life (ax for age 0), and is calculated based on the mortality rate during the first year of life (m0). + + There is a slight different procedure for male or female. + + More details: https://www.rdocumentation.org/packages/MortHump/versions/0.2/topics/AKm02a0 + """ + if is_male: + return np.where(m0 < 0.0230, 0.14929 - 1.99545 * m0, np.where(m0 < 0.08307, 0.02832 + 3.26201 * m0, 0.29915)) + else: + return np.where(m0 < 0.01724, 0.14903 - 2.05527 * m0, np.where(m0 < 0.06891, 0.04667 + 3.88089 * m0, 0.31411)) + + +def gini_from_mx(tb_group: Table) -> Variable: + """Get Gini coefficient from central death rate. + + This code is adapted from the original R code: https://github.com/jmaburto/Dynamics_Code/tree/V1.0/R%20code + """ + # Get values from input + mx = tb_group["central_death_rate"].values + is_male = tb_group.name[2] == "male" + + # Estimate i_openage, ax + i_openage = len(mx) + m0 = cast(float, mx[0]) + ax = np.full_like(mx, 0.5) + ax[0] = AKm02a0(m0=m0, is_male=is_male) + ax[i_openage - 1] = 1 / mx[i_openage - 1] # type: ignore + + # Estimate X_ + age = np.arange(i_openage) + ax + e = np.ones_like(age) + X_ = np.abs(np.outer(e, age) - np.outer(age, e)) + + # Estimate D + OPENAGE = i_openage - 1 + ## Calculates the probability of dying in each age interval + qx = mx / (1 + (1 - ax) * mx) # type: ignore + qx[i_openage - 1] = 1 if not np.isnan(qx[i_openage - 1]) else np.nan + ## Probability of surviving in each age interval + px = 1 - qx + px[np.isnan(px)] = 0 + ## number of survivors at the start of each interval + RADIX = 1 # starting value + lx = np.concatenate(([RADIX], RADIX * np.cumprod(px[:OPENAGE]))) + ## number of people who die in each interval + dx = lx * qx + ## number of person years lived in each interval + ## [number of initial survivors in that interval] - (1 - [number of years lived during that interval]) * [number who die in the interval] + Lx = lx - (1 - ax) * dx + Lx[i_openage - 1] = lx[i_openage - 1] * ax[i_openage - 1] + ## total number of life years from a given age to the end of the cohort + Tx = np.concatenate((np.cumsum(Lx[:OPENAGE][::-1])[::-1], [0])) + Lx[i_openage - 1] + ## life expectancy + ex = Tx / lx + ## matrix with the number of deaths for each age-pair combination + D = np.outer(dx, dx) + + # Estimate Gini + ## total inequality in lifespans: sum of the product of the matrix D by the age difference, np.sum(D * X_) + ## divided by the life expectancy at birth x2 (helps to normalise it to a number between 0 and 1) + G = np.sum(D * X_) / (2 * ex[0]) + + var = Variable({"life_expectancy_gini": G}) + return var diff --git a/etl/steps/data/garden/demography/2024-12-03/life_expectancy.meta.yml b/etl/steps/data/garden/demography/2024-12-03/life_expectancy.meta.yml new file mode 100644 index 00000000000..c22b9206970 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-03/life_expectancy.meta.yml @@ -0,0 +1,166 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + global: + selected_entities: + with_americas: &sel_entities_americas + - Africa + - Americas + - Asia + - Europe + - Oceania + - World + standard: &sel_entities + - Africa + - Northern America + - Latin America and the Caribbean + - Asia + - Europe + - Oceania + - World + title_base: Period life expectancy + title_public: Life expectancy + data_used: |- + <%- if (age == 0) and (sex == 'total') -%> + Prior to 1950, we use HMD (2024) data combined with Zijdeman (2015). From 1950 onwards, we use UN WPP (2024) data. For old regional data, we use Riley (2005) estimates. + <%- else -%> + Prior to 1950, we use HMD (2024) data. From 1950 onwards, we use UN WPP (2024) data. + <%- endif -%> + point_1: |- + Period life expectancy is a metric that summarizes death rates across all age groups in one particular year. + point_2: |- + <%- if age == '0' -%> + For a given year, it represents the average lifespan for a hypothetical group of people, if they experienced the same age-specific death rates throughout their whole lives as the age-specific death rates seen in that particular year. + <%- else -%> + For a given year, it represents the remaining average lifespan for a hypothetical group of people, if they experienced the same age-specific death rates throughout the rest of their lives as the age-specific death rates seen in that particular year. + <%- endif -%> + + common: + description_short: |- + <%- if age == 0 -%> + <%- if sex == 'total' -%> + The period life expectancy at birth, in a given year. + <%- else -%> + The period life expectancy at birth among << sex + 's' >>, in a given year. + <%- endif -%> + <%- else -%> + <%- if sex == 'total' -%> + The total period life expectancy at age << age >>, in a given year. + <%- else -%> + The total period life expectancy at age << age >> among << sex + 's' >>, in a given year. + <%- endif -%> + <%- endif -%> + description_key: + - |- + {definitions.global.point_1} + - |- + {definitions.global.point_2} + - |- + {definitions.global.data_used} + presentation: + title_public: |- + {definitions.global.title_public} at << age if age != 0 else 'birth'>> + attribution_short: HMD, UN WPP + topic_tags: + - Life Expectancy + grapher_config: + hasMapTab: true + selectedEntityNames: *sel_entities + display: + numDecimalPlaces: 1 + unit: years + short_unit: years + processing_level: minor + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/dataset/ +dataset: + title: Life Expectancy (period) + update_period_days: 365 + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/tables/ +tables: + # HISTORICAL VALUES (until today) + life_expectancy: + variables: + # {definitions.global.title_base} + life_expectancy: + title: |- + {definitions.global.title_base} + display: &display_hist + numDecimalPlaces: 1 + name: |- + {definitions.global.title_public} at << 'birth' if (age == 0) else age >><< ', ' + sex + 's' if (sex != 'total') >> + presentation: + title_variant: &title_variant_hist << sex + 's, ' if sex != 'total' >>period tables + + life_expectancy_0: + title: |- + {definitions.global.title_base} at birth + display: *display_hist + presentation: + attribution_short: Various sources + title_variant: *title_variant_hist + grapher_config: + selectedEntityNames: *sel_entities_americas + attribution: |- + UN WPP (2024); HMD (2024); Zijdeman et al. (2015); Riley (2005) + + # WITH PROJECTIONS + life_expectancy_with_proj: + variables: + life_expectancy_with_proj: + title: |- + {definitions.global.title_base} (with projections) + display: &display_w_proj + numDecimalPlaces: 1 + name: |- + {definitions.global.title_public} at << 'birth' if (age == 0) else age >><< ', ' + sex + 's' if (sex != 'total') >>, with UN medium projections + presentation: + title_variant: &title_variant_w_proj << sex + 's, ' if sex != 'total' >>period tables, with UN medium projections + + life_expectancy_0_with_proj: + title: |- + {definitions.global.title_base} at birth (with projections) + display: *display_w_proj + presentation: + attribution_short: Various sources + title_variant: *title_variant_w_proj + grapher_config: + selectedEntityNames: *sel_entities_americas + attribution: |- + UN WPP (2024); HMD (2024); Zijdeman et al. (2015); Riley (2005) + + # ONLY PROJECTIONS + life_expectancy_only_proj: + variables: + life_expectancy_only_proj: + title: |- + {definitions.global.title_base} (only projections) + description_key: + - |- + {definitions.global.point_1} + - |- + {definitions.global.point_2} + display: &display_o_proj + numDecimalPlaces: 1 + name: |- + {definitions.global.title_public} at << 'birth' if (age == 0) else age >><< ', ' + sex + 's' if (sex != 'total') >>, medium projection + presentation: + attribution_short: &attr_o_proj UN WPP + title_variant: &title_variant_o_proj << sex + 's, ' if sex != 'total' >>period tables, medium projection + + life_expectancy_0_only_proj: + title: |- + {definitions.global.title_base} at birth (only projections) + description_key: + - |- + {definitions.global.point_1} + - |- + {definitions.global.point_2} + display: *display_o_proj + presentation: + attribution_short: *attr_o_proj + title_variant: *title_variant_o_proj + grapher_config: + selectedEntityNames: *sel_entities_americas diff --git a/etl/steps/data/garden/demography/2024-12-03/life_expectancy.py b/etl/steps/data/garden/demography/2024-12-03/life_expectancy.py new file mode 100644 index 00000000000..322f8af2631 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-03/life_expectancy.py @@ -0,0 +1,368 @@ +"""Load a meadow dataset and create a garden dataset.""" + +import owid.catalog.processing as pr +from owid.catalog import Dataset, Table + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) +# Year of last estimate +YEAR_ESTIMATE_LAST = 2023 +YEAR_WPP_START = 1950 + +# Region mapping +# We will be using continent names without (Entity) suffix. This way charts show continuity between lines from different datasets (e.g. riley and UN) +REGION_MAPPING = { + "Africa (Riley 2005)": "Africa", + "Americas (Riley 2005)": "Americas", + "Asia (Riley 2005)": "Asia", + "Europe (Riley 2005)": "Europe", + "Oceania (Riley 2005)": "Oceania", + "Africa (UN)": "Africa", + "Northern America (UN)": "Northern America", + "Latin America and the Caribbean (UN)": "Latin America and the Caribbean", + "Asia (UN)": "Asia", + "Europe (UN)": "Europe", + "Oceania (UN)": "Oceania", +} + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + ## Life tables + paths.log.info("reading dataset `life_tables`") + ds_lt = paths.load_dataset("life_tables") + tb_lt = ds_lt.read("life_tables") + ## zijdeman_et_al_2015 + paths.log.info("reading dataset `zijdeman_et_al_2015`") + ds_zi = paths.load_dataset("zijdeman_et_al_2015") + tb_zi = ds_zi.read("zijdeman_et_al_2015") + ## Riley + paths.log.info("reading dataset `riley_2005`") + ds_ri = paths.load_dataset("riley_2005") + tb_ri = ds_ri.read("riley_2005") + ## WPP + paths.log.info("reading dataset `un_wpp`") + ds_un = paths.load_dataset("un_wpp") + tb_un = ds_un.read("life_expectancy") + + # + # Process data. + # + paths.log.info("processing data") + tb_lt = process_lt(tb_lt) + tb_un = process_un(tb_un) + tb_zi = process_zi(tb_zi) + tb_ri = process_ri(tb_ri) + + paths.log.info("combining tables") + tb = combine_tables(tb_lt, tb_un, tb_zi, tb_ri) + + # Rename regions, and use column 'country' instead of 'country' + tb["country"] = tb["country"].replace(REGION_MAPPING) + + # Add Americas + # tb = add_americas(tb, ds_un) + + ## Check values + paths.log.info("final checks") + _check_column_values(tb, "sex", {"total", "male", "female"}) + _check_column_values(tb, "age", {0, 10, 15, 25, 45, 65, 80}) + + # Create three tables: (i) only historical values, (ii) only future values, (iii) all values + columns_index = ["country", "year", "sex", "age"] + + ## (i) Main table (historical values) + tb_main = tb.loc[tb["year"] <= YEAR_ESTIMATE_LAST].copy() + + ## (ii) Only projections + tb_only_proj = tb.loc[tb["year"] > YEAR_ESTIMATE_LAST].copy() + tb_only_proj = _add_suffix_to_indicators(tb_only_proj, "_only_proj", columns_index=columns_index) + ## Table only with projections should only contain UN as origin + origins_un = [origin for origin in tb_main["life_expectancy"].m.origins if origin.producer == "United Nations"] + for col in tb_only_proj.columns: + tb_only_proj[col].origins = origins_un + + ## (iii) All values + tb_with_proj = tb.copy() + # Only preserve ages that have projections (i.e. data after YEAR_ESTIMATE_LAST) + ages_with_projections = set(tb_with_proj.loc[tb_with_proj["year"] > YEAR_ESTIMATE_LAST, "age"].unique()) + tb_with_proj = tb_with_proj.loc[tb_with_proj["age"].isin(ages_with_projections)] + # Column names + tb_with_proj = _add_suffix_to_indicators(tb_with_proj, "_with_proj", columns_index=columns_index) + + # Format + tables = [ + tb_main.format(columns_index, short_name=paths.short_name), + tb_only_proj.format(columns_index, short_name=f"{paths.short_name}_only_proj"), + tb_with_proj.format(columns_index, short_name=f"{paths.short_name}_with_proj"), + ] + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset(dest_dir, tables=tables, check_variables_metadata=True, default_metadata=ds_lt.metadata) + + # Save changes in the new garden dataset. + ds_garden.save() + + +def _add_suffix_to_indicators(tb, suffix, columns_index=None): + if columns_index is None: + columns_index = [] + tb.columns = [f"{col}{suffix}" if col not in columns_index else col for col in tb.columns] + return tb + + +def process_lt(tb: Table) -> Table: + """Process LT data and output it in the desired format. + + Desired format is with columns country, year, sex, age | life_expectancy. + """ + tb = tb.loc[ + (tb["age"].isin(["0", "10", "15", "25", "45", "65", "80"])) & (tb["type"] == "period"), + ["country", "year", "sex", "age", "life_expectancy"], + ] + + # Assign dtype + tb["age"] = tb["age"].astype("Int64") + + # Update life_expectancy values + tb["life_expectancy"] = tb["life_expectancy"] + tb["age"] + + # Check latest year + assert ( + tb["year"].max() == YEAR_ESTIMATE_LAST + ), f"Last year was {tb['year'].max()}, but should be {YEAR_ESTIMATE_LAST}" + + # Check column values + ## sex + _check_column_values(tb, "sex", {"total", "female", "male"}) + ## age + _check_column_values(tb, "age", {0, 10, 15, 25, 45, 65, 80}) + + return tb + + +def process_un(tb: Table) -> Table: + """Process UN WPP data and output it in the desired format. + + Desired format is with columns country, year, sex, age | life_expectancy. + """ + # Sanity check + assert ( + tb["year"].min() == YEAR_WPP_START + ), f"Year of first estimate is different than {YEAR_WPP_START}, it is {tb['year'].min()}" + + # Filter + ## dimension values: metric=life_expectancy, variant=medium, year >= YEAR_ESTIMATE_LAST + ## columns: country, year, value, sex, age + tb = tb.loc[ + (tb["year"] > YEAR_ESTIMATE_LAST) & (tb["variant"] == "medium"), + ["country", "year", "sex", "age", "life_expectancy"], + ] + + # Rename column values + tb["age"] = tb["age"].replace({"at birth": "0"}).astype("Int64") + tb["sex"] = tb["sex"].replace({"all": "total"}) + + # Check column values + ## sex + _check_column_values(tb, "sex", {"total", "female", "male"}) + ## age + _check_column_values(tb, "age", {0, 15, 65, 80}) + + # Check minimum year + assert ( + tb.groupby("country", observed=True).year.min() == YEAR_ESTIMATE_LAST + 1 + ).all(), f"Some entry with latest year different than {YEAR_ESTIMATE_LAST}" + + return tb + + +def process_zi(tb: Table) -> Table: + """Process Zijdeman data and output it in the desired format. + + Desired format is with columns country, year, sex, age | life_expectancy. + """ + # Filter + ## dimension values: metric=life_expectancy, variant=medium, year >= YEAR_ESTIMATE_LAST + ## columns: country, year, value, sex, age + tb = tb.loc[(tb["year"] <= YEAR_ESTIMATE_LAST)] + + # Add columns + # tb["type"] = "period" + tb["age"] = 0 + tb["sex"] = "total" + + # Resolution + tb["life_expectancy"] = tb["life_expectancy"].astype("Float64").round(3) + + # Dtypes + tb = tb.astype( + { + "age": "Int64", + "sex": "string", + } + ) + + # Sanity check + assert tb["year"].max() == 2012, f"Last year was {tb['year'].max()}, but should be 2012" + + return tb + + +def process_ri(tb: Table) -> Table: + """Process Riley data and output it in the desired format. + + Desired format is with columns country, year, sex, age | life_expectancy. + """ + # Filter + ## dimension values: metric=life_expectancy, variant=medium, year >= YEAR_ESTIMATE_LAST + ## columns: country, year, value, sex, age + tb = tb.loc[(tb["year"] < 1950),] + + # Rename column names + tb = tb.rename(columns={"entity": "country"}) + + # Add columns + # tb["type"] = "period" + tb["sex"] = "total" + tb["age"] = 0 + + # Dtypes + tb = tb.astype( + { + "age": "Int64", + "sex": "string", + } + ) + + # Resolution + tb["life_expectancy"] = tb["life_expectancy"].astype("Float64").round(3) + + return tb + + +def combine_tables(tb_lt: Table, tb_un: Table, tb_zi: Table, tb_ri: Table) -> Table: + """Combine all LE tables. + + - Only HMD (within LT) contains cohort data. + - LE broken down by sex and age is available from LT and UN_WPP. + - LT already contains UN_WPP data, but without projections. That's why we also use UN WPP's + - RIL and ZIJ contain figures for all sexes and at birth. Only period. + """ + tb = pr.concat([tb_lt, tb_un], ignore_index=True, short_name="life_expectancy") + + # Separate LE at birth from at different ages + mask = (tb["age"] == 0) & (tb["sex"] == "total") + tb_0 = tb.loc[mask] + tb = tb.loc[~mask] + + # Extend tb_0 (only for period) + ## Zijdeman: complement country data + tb_0 = tb_0.merge(tb_zi, how="outer", on=["country", "year", "sex", "age"], suffixes=("", "_zij")) + tb_0["life_expectancy"] = tb_0["life_expectancy"].fillna(tb_0["life_expectancy_zij"]) + tb_0 = tb_0.drop(columns=["life_expectancy_zij"]) + ## Riley: complement with continent data + tb_0 = pr.concat([tb_0, tb_ri], ignore_index=True) + + # Combine tb_0 with tb + tb = tb.merge(tb_0, on=["country", "year", "sex", "age"], how="outer", suffixes=("", "_0")) + + # For some reason, 'sex' is assigned type object + tb["sex"] = tb["sex"].astype("string") + + return tb + + +def _check_column_values(tb: Table, column: str, expected_values: set) -> None: + """Check that a column has only expected values.""" + unexpected_values = set(tb[column]) - expected_values + assert not unexpected_values, f"Unexpected values found in column {column}: {unexpected_values}" + + +def add_americas(tb: Table, ds_population: Dataset) -> Table: + """Estimate value for the Americas using North America and LATAM/Caribbean. + + Only performs this estimation for: + + sex = all + age = 0 + + It estimates it by doing the population-weighted average of life expectancies. + """ + # filter only member countries of the region + AMERICAS_MEMBERS = ["Northern America", "Latin America and the Caribbean"] + tb_am = tb.loc[(tb["country"].isin(AMERICAS_MEMBERS)) & (tb["sex"] == "total") & (tb["age"] == 0),].copy() + + # sanity check + assert ( + tb_am.groupby(["country", "year"]).size().max() == 1 + ), "There is more than one entry for a (country, year) tuple!" + + # add population for LATAM and Northern America (from WPP, hence since 1950) + assert tb_am["year"].min() == YEAR_WPP_START + tb_am = add_population_americas_from_wpp(tb_am, ds_population) + + # sanity check: ensure there are NO missing values. This way, we can safely do the groupby + assert (tb_am[["life_expectancy_0", "population"]].isna().sum() == 0).all() + + # estimate values for regions + # y(country) = weight(country) * metric(country) + tb_am["life_expectancy_0"] *= tb_am["population"] + + # z(region) = sum{ y(country) } for country in region + tb_am = tb_am.groupby("year", as_index=False)[["life_expectancy_0", "population"]].sum() + + # z(region) / sum{ population(country) } for country in region + tb_am["life_expectancy_0"] /= tb_am["population"] + + # assign region name + tb_am = tb_am.assign( + country="Americas", + sex="total", + age=0, + ) + + # drop unused column + tb_am = tb_am.drop(columns="population") + + # concatenate + tb = pr.concat([tb, tb_am], ignore_index=True) + return tb + + +def add_population_americas_from_wpp(tb: Table, ds_population: Dataset) -> Table: + """Add population values for LATAM and Northern America. + + Data is sourced from UN WPP, hence only available since 1950. + """ + pop = load_america_population_from_unwpp(ds_population) + tb = tb.merge(pop, on=["country", "year"]) + return tb + + +def load_america_population_from_unwpp(ds_population: Dataset) -> Table: + """Load population data from UN WPP for Northern America and Latin America and the Caribbean. + + We use this dataset instead of the long-run because we want the entities as defined by the UN. + """ + # load population from WPP + countries = ["Latin America and the Caribbean (UN)", "Northern America (UN)"] + tb = ds_population.read("population") + tb = tb.loc[ + (tb["country"].isin(countries)) + & (tb["sex"] == "all") + & (tb["age"] == "all") + & (tb["variant"].isin(["estimates", "medium"])), + ["country", "year", "population"], + ] + assert len(set(tb["country"])) == 2, f"Check that all of {countries} are in df" + tb["country"] = tb["country"].replace(REGION_MAPPING).drop(columns="country") + + return tb diff --git a/etl/steps/data/garden/demography/2024-12-03/life_tables.meta.yml b/etl/steps/data/garden/demography/2024-12-03/life_tables.meta.yml new file mode 100644 index 00000000000..a5de8e6bd4b --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-03/life_tables.meta.yml @@ -0,0 +1,223 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + grapher_config: + selectedEntityNames: + - Italy + - England and Wales + - France + - Sweden + topic_tags: + - Life Expectancy + + global: + life_expectancy: + point_1: |- + <% if type == "period" %> + Period life expectancy is a metric that summarizes death rates across all age groups in one particular year. + <%- else %> + Cohort life expectancy is the average lifespan of a group of people, usually a birth cohort – people born in the same year. + <%- endif %> + point_2: |- + <% if type == "period" %> + <%- if age == '0' %> + For a given year, it represents the average lifespan for a hypothetical group of people, if they experienced the same age-specific death rates throughout their whole lives as the age-specific death rates seen in that particular year. + <%- else %> + For a given year, it represents the remaining average lifespan for a hypothetical group of people, if they experienced the same age-specific death rates throughout the rest of their lives as the age-specific death rates seen in that particular year. + <%- endif %> + <%- else %> + <%- if age == '0' %> + It is calculated by tracking individuals from that cohort throughout their lives until death, and calculating their average lifespan. + <%- else %> + It is calculated by tracking individuals from that cohort throughout the rest of their lives until death, and calculating their average remaining lifespan. + <%- endif %> + <%- endif %> + data_used: |- + Prior to 1950, we use HMD (2024) data. From 1950 onwards, we use UN WPP (2024) data. + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/dataset/ +dataset: + title: Life tables + update_period_days: 365 + description: |- + Life tables from UN and HMD. + + For period data, data prior to 1950 is from HMD, and data from 1950 onwards is from UN WPP. + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/tables/ +tables: + life_tables: + variables: + central_death_rate: + title: Central death rate + unit: deaths per 1,000 people + description_short: |- + The death rate, calculated as the number of deaths divided by the average number of people alive during the interval. + description_key: + - |- + The death rate is measured using the number of person-years lived during the interval. + - |- + Person-years refers to the combined total time that a group of people has lived. For example, if 10 people each live for 2 years, they collectively contribute 20 person-years. + - |- + The death rate is slightly different from the 'probability of death' during the interval, because the 'probability of death' metric uses a different denominator: the number of people alive at that age at the start of the interval, while this indicator uses the average number of people alive during the interval. + - |- + {definitions.global.data_used} + description_processing: |- + The original metric is given as a fraction between 0 and 1 (i.e. per-capita). We multiply this by 1,000 to get a per-1,000 people rate. + processing_level: minor + display: + name: |- + Central death rate at << 'birth' if (age == '0') else age >><< ', ' + sex + 's' if (sex != 'both') >>, << type >> + presentation: + title_public: Central death rate at << age if age != '0' else 'birth'>> + title_variant: << sex + 's, ' if sex != 'both' >><< type + ' tables'>> + topic_tags: + - Life Expectancy + + probability_of_death: + title: Probability of death + unit: "%" + description_short: |- + The probability of dying in a given interval, among people who survived to the start of that interval. + description_key: + - |- + For example, the probability of death for a 50 year old in a given year is found by: dividing the number of deaths in 50 year olds that year, by the number of people alive at the age of 50 at the start of the year. + description_processing: |- + The original metric is given as a fraction between 0 and 1 (i.e. per-capita). We multiply this by 100 to get a percentage. + processing_level: minor + display: + name: |- + Probability of death at << 'birth' if (age == '0') else age >><< ', ' + sex + 's' if (sex != 'both') >>, << type >> + presentation: + title_public: Probability of death at << age if age != '0' else 'birth'>> + title_variant: << sex + 's, ' if sex != 'both' >><< type + ' tables'>> + topic_tags: + - Life Expectancy + - Causes of Death + + average_survival_length: + title: Average survival length + short_unit: years + unit: years + description_short: Average length of survival between ages x and x+n for persons dying in the interval. + + number_survivors: + title: Number of survivors + unit: survivors + description_short: Number of survivors at a given age, assuming survivors at 0 years old is 100,000. + + number_deaths: + title: Number of deaths + short_unit: deaths + unit: deaths + description_short: Number of deaths during a given age. + presentation: + topic_tags: + - Life Expectancy + - Causes of Death + + number_person_years_lived: + title: Number of person-years lived + unit: person-years + description_short: Number of person-years lived between a given ages and a year after. + + number_person_years_remaining: + title: Number of person-years remaining + unit: person-years + description_short: Number of person-years remaining after a given age. + + life_expectancy: + title: Life expectancy + short_unit: years + unit: years + description_short: |- + <%- if age == '0' -%> + <%- if sex == 'both' -%> + The << type >> life expectancy at birth, in a given year. + <%- else -%> + The << type >> life expectancy at birth among << sex + 's' >>, in a given year. + <%- endif -%> + <%- else -%> + <%- if sex == 'both' -%> + The remaining << type >> life expectancy at age << age >>, in a given year. + <%- else -%> + The remaining << type >> life expectancy at age << age >> among << sex + 's' >>, in a given year. + <%- endif -%> + <%- endif -%> + description_key: + - |- + {definitions.global.life_expectancy.point_1} + - |- + {definitions.global.life_expectancy.point_2} + - |- + <%- if age != '0' -%> + <%- if type == "period" -%> + This shows the remaining life expectancy among people who have already reached the age << age >>, using death rates from their age group and older age groups. + <%- else -%> + This shows the remaining cohort life expectancy of people who have reached the age << age >>. + <%- endif -%> + <%- endif -%> + - |- + {definitions.global.data_used} + display: + numDecimalPlaces: 1 + name: |- + Life expectancy at << 'birth' if (age == '0') else age >><< ', ' + sex + 's' if (sex != 'both') >>, << type >> + presentation: + title_public: Life expectancy at << age if age != '0' else 'birth'>> + title_variant: << sex + 's, ' if sex != 'both' >><< type + ' tables'>> + + diff_ratios: + variables: + life_expectancy_fm_diff: + title: Life expectancy difference (f-m) + short_unit: years + unit: years + description_short: |- + The difference in << type >> life expectancy (females - males) at a given age. + description_key: + - Higher values indicate longer life expectancy among females than males. + - |- + {definitions.global.life_expectancy.point_1} + - |- + {definitions.global.life_expectancy.point_2} + - |- + {definitions.global.data_used} + display: + numDecimalPlaces: 1 + name: |- + Life expectancy (female-male difference) at << 'birth' if (age == '0') else age >>, << type >> + presentation: + title_public: Life expectancy at << age if age != '0' else 'birth'>> + title_variant: female-male difference, << type + ' tables'>> + topic_tags: + - Life Expectancy + - Gender Ratio + + life_expectancy_fm_ratio: + title: Life expectancy ratio (f/m) + unit: "" + short_unit: "" + description_short: |- + The ratio of << type >> life expectancy (females/males) at a given age. + description_key: + - Higher values indicate longer life expectancy among females than males. + - |- + {definitions.global.life_expectancy.point_1} + - |- + {definitions.global.life_expectancy.point_2} + - |- + {definitions.global.data_used} + display: + numDecimalPlaces: 1 + name: |- + Life expectancy (female-to-male ratio) at << 'birth' if (age == '0') else age >>, << type >> + presentation: + title_public: Life expectancy at << age if age != '0' else 'birth'>> + title_variant: female-to-male ratio, << type + ' tables'>> + topic_tags: + - Life Expectancy + - Gender Ratio diff --git a/etl/steps/data/garden/demography/2024-12-03/life_tables.py b/etl/steps/data/garden/demography/2024-12-03/life_tables.py new file mode 100644 index 00000000000..33fe750122c --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-03/life_tables.py @@ -0,0 +1,194 @@ +"""Load a meadow dataset and create a garden dataset. + + +Combines HMD and UN life tables. + +Some notes: + + - Time coverage: + - UN contains data on many more countries, but only since 1950. + - HMD contains data on fewer countries, but since 1676! + - We therefore use UN since 1950 for all countries, and HMD prior to that. We use the same source for all countries in each time period to ensure comparability across countries. + - Age groups: + - HMD contains single-age groups from 0 to 109 and 110+ (equivalent to >=110). It also contains data on wider age groups, but we discard these. + - UN contains single-age groups from 0 to 99 and 100+ (equivalent to >=100) +""" + +import numpy as np +import owid.catalog.processing as pr +from owid.catalog import Table + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) +# List of indicator columns +COLUMNS_INDICATORS = [ + "central_death_rate", + "probability_of_death", + "probability_of_survival", + "number_survivors", + "number_deaths", + "number_person_years_lived", + "survivorship_ratio", + "number_person_years_remaining", + "life_expectancy", + "average_survival_length", +] +COLUMN_INDICATORS_REL = [ + "life_expectancy_fm_diff", + "life_expectancy_fm_ratio", + "central_death_rate_mf_ratio", +] +COLUMNS_INDEX = [ + "country", + "year", + "sex", + "age", + "type", +] +COLUMNS_INDEX_REL = [ + "country", + "year", + "age", + "type", +] + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow datasets. + paths.log.info("load dataset, tables") + ds_hmd = paths.load_dataset("hmd") + ds_un = paths.load_dataset("un_wpp_lt") + + # Read table from meadow dataset. + tb_hmd = ds_hmd.read("life_tables") + tb_hmd_diff = ds_hmd.read("diff_ratios") + tb_un = ds_un.read("un_wpp_lt") + + # + # Process data. + # + tb_un = tb_un.rename( + columns={ + "location": "country", + } + ) + # Set type='period' for UN data + tb_un["type"] = "period" + + # Keep only single-years + ## Get only single-year, set dtype as int + flag = ~tb_hmd["age"].str.contains("-") + tb_hmd = tb_hmd.loc[flag] + flag = ~tb_hmd_diff["age"].str.contains("-") + tb_hmd_diff = tb_hmd_diff.loc[flag] + + # Add life expectancy differences and ratios + paths.log.info("calculating extra variables (ratio and difference in life expectancy for f and m).") + tb_un_rel = make_table_diffs_ratios(tb_un) + + # Combine HMD + UN + paths.log.info("concatenate tables") + tb = combine_tables(tb_hmd, tb_un, COLUMNS_INDEX, COLUMNS_INDICATORS) + tb_rel = combine_tables(tb_hmd_diff, tb_un_rel, COLUMNS_INDEX_REL, COLUMN_INDICATORS_REL) + + # Set DTypes + dtypes = { + "type": "string", + } + tb = tb.astype(dtypes) + tb_rel = tb_rel.astype(dtypes) + + # Set index + tb = tb.format(COLUMNS_INDEX, short_name=paths.short_name) + tb_rel = tb_rel.format(COLUMNS_INDEX_REL, short_name="diff_ratios") + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset(dest_dir, tables=[tb, tb_rel], check_variables_metadata=True) + + # Save changes in the new garden dataset. + ds_garden.save() + + +def combine_tables(tb_hmd: Table, tb_un: Table, cols_index, cols_indicators) -> Table: + """Combine HMD and UN life tables. + + - UN only provides period data. + - We use UN data after 1950. Prior to that, we use HMD. + - We considered using HMD over UN after 1950 if data was available for a given country for all years, ages and sexes. + - However, this is only the case for very few countries: Australia, Germany, Hungary, Lithuania, Northern Ireland, Scotland, United Kingdom. + - We decided against this to ensure comparability across countries (i.e. all countries use same source after 1950). + """ + # HMD + ## Sanity check years + assert tb_hmd["year"].max() == 2023, "HMD data should end in 2023" + assert tb_hmd["year"].min() == 1751, "HMD data should start in 1751" + ## Keep only period HMD data prior to 1950 (UN data starts in 1950) + tb_hmd = tb_hmd.loc[((tb_hmd["year"] < 1950) & (tb_hmd["type"] == "period")) | (tb_hmd["type"] == "cohort")] + ## Filter relevant columns (UN has two columns that HMD doesn't: 'probability_of_survival', 'survivorship_ratio') + columns_indicators_hmd = [col for col in tb_hmd.columns if col in cols_indicators] + tb_hmd = tb_hmd.loc[:, cols_index + columns_indicators_hmd] + + # UN + ## Sanity check years + assert tb_un["year"].max() == 2023, "UN data should end in 2023" + assert tb_un["year"].min() == 1950, "UN data should start in 1950" + assert (tb_un["year"].drop_duplicates().diff().dropna() == 1).all(), "UN data should be yearly" + ## Filter relevant columns + tb_un = tb_un.loc[:, cols_index + cols_indicators] + + # Combine tables + tb = pr.concat([tb_hmd, tb_un], short_name=paths.short_name) + + # Remove all-NaN rows + tb = tb.dropna(subset=cols_indicators, how="all") + + return tb + + +def make_table_diffs_ratios(tb: Table) -> Table: + """Create table with metric differences and ratios. + + Currently, we estimate: + + - female - male: Life expectancy + - male/female: Life Expectancy, Central Death Rate + """ + # Pivot & obtain differences and ratios + cols_index = ["country", "year", "age", "type"] + tb_new = ( + tb.pivot_table( + index=cols_index, + columns="sex", + values=["life_expectancy", "central_death_rate"], + ) + .assign( + life_expectancy_fm_diff=lambda df: df[("life_expectancy", "female")] - df[("life_expectancy", "male")], + life_expectancy_fm_ratio=lambda df: df[("life_expectancy", "female")] / df[("life_expectancy", "male")], + central_death_rate_mf_ratio=lambda df: df[("central_death_rate", "male")] + / df[("central_death_rate", "female")], + ) + .reset_index() + ) + + # Keep relevant columns + cols = [col for col in tb_new.columns if col[1] == ""] + tb_new = tb_new.loc[:, cols] + + # Rename columns + tb_new.columns = [col[0] for col in tb_new.columns] + + # Add metadata back + for col in tb_new.columns: + if col not in cols_index: + tb_new[col].metadata.origins = tb["life_expectancy"].m.origins.copy() + tb_new[col] = tb_new[col].replace([np.inf, -np.inf], np.nan) + + return tb_new diff --git a/etl/steps/data/garden/demography/2024-12-03/phi_gender_le.meta.yml b/etl/steps/data/garden/demography/2024-12-03/phi_gender_le.meta.yml new file mode 100644 index 00000000000..424fe96f873 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-03/phi_gender_le.meta.yml @@ -0,0 +1,40 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Life Expectancy + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/dataset/ +dataset: + update_period_days: 365 + title: Outsurvival statistic + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/tables/ +tables: + phi_gender_le: + variables: + phi: + title: Outsurvival statistic at birth + unit: "%" + short_unit: "%" + description_short: |- + The probability that a male will live longer than a female if both are randomly selected from the population at birth. + description_processing: |- + This was calculated using scripts from Bergeron-Boucher et al. (2022). + + Citation: Bergeron-Boucher, M.-P., Alvarez, J.-A., Kashnitsky, I., & Zarulli, V. (2022). Probability of males to outlive females: An international comparison from 1751 to 2020. BMJ Open, 12(8), e059964. https://doi.org/10.1136/bmjopen-2021-059964 + + Code available at: https://github.com/CPop-SDU/outsurvival-in-perspective + description_key: + - The probability that a male will live longer than a female, in a given population during a given interval. + - This is calculated for random pairs of one male and one female at age 0. + display: + numDecimalPlaces: 1 + presentation: + grapher_config: + hasMapTab: true + attribution: |- + Human Mortality Database (2024); UN, World Population Prospects (2024); Bergeron-Boucher et al. (2022) diff --git a/etl/steps/data/garden/demography/2024-12-03/phi_gender_le.py b/etl/steps/data/garden/demography/2024-12-03/phi_gender_le.py new file mode 100644 index 00000000000..a8acfd9f94c --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-03/phi_gender_le.py @@ -0,0 +1,106 @@ +"""Load a meadow dataset and create a garden dataset.""" + +from owid.catalog import Table + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("life_tables") + + # Read table from meadow dataset. + tb = ds_meadow.read("life_tables") + + # + # Process data. + # + paths.log.info("replace 110+ -> 110, 100+ -> 100") + tb["age"] = ( + tb["age"] + .replace( + { + "110+": "110", + "100+": "100", + } + ) + .astype(int) + ) + + # Keep only period data broken down by sex + paths.log.info("keep only type='period' and sex in {'male', 'female'}") + tb = tb.loc[(tb["type"] == "period") & (tb["sex"].isin(["female", "male"]))].drop(columns=["type"]) + + # Add phi + paths.log.info("add phi parameter") + tb = make_table_phi(tb) + + # Set index + tb = tb.format(["country", "year"], short_name=paths.short_name) + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata + ) + + # Save changes in the new garden dataset. + ds_garden.save() + + +def make_table_phi(tb: Table) -> Table: + """Estimate phi. + + Phi is defined as the outsurvival probability of males (i.e. probability that a male will live longer than a female in a given population). + + This is estimated using Equation 2 from https://bmjopen.bmj.com/content/bmjopen/12/8/e059964.full.pdf. + + Inspired by code: + - https://github.com/CPop-SDU/sex-gap-e0-pnas/tree/main + - https://github.com/CPop-SDU/outsurvival-in-perspective + """ + # Copy original metadata + origins = tb["number_deaths"].metadata.origins + + # Calculate standard deviations + tb["number_survivors"] = tb["number_survivors"] / 1e5 + tb["number_deaths"] = tb["number_deaths"] / 1e5 + + # Pivot table to align males and females for the different metrics + tb = tb.pivot( + index=["country", "year", "age"], columns="sex", values=["number_survivors", "number_deaths"] + ).reset_index() + + # Order + tb = tb.sort_values(["country", "year", "age"]) + + # Shift one up (note the subindex in the equation 'x-n', in our case n=1 (age group width)) + column = ("number_survivors", "male") + tb[column] = tb.groupby(["country", "year"])[[column]].shift(-1).squeeze() + + # Estimate phi_i (i.e. Eq 2 for a specific age group, without the summation) + tb["phi"] = ( + tb["number_deaths"]["female"] * tb["number_survivors"]["male"] + + tb["number_deaths"]["female"] * tb["number_deaths"]["male"] / 2 + ) + # Apply the summation from Eq 2 + tb = tb.groupby(["country", "year"], as_index=False, observed=True)[[("phi", "")]].sum() + + # Scale + tb["phi"] = (tb["phi"] * 100).round(2) + + # Fix column names (remove multiindex) + tb.columns = [col[0] for col in tb.columns] + + # Copy metadata + tb["phi"].metadata.origins = origins + + return tb diff --git a/etl/steps/data/garden/demography/2024-12-05/paternal_ages.countries.json b/etl/steps/data/garden/demography/2024-12-05/paternal_ages.countries.json new file mode 100644 index 00000000000..f780eab6675 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-05/paternal_ages.countries.json @@ -0,0 +1,133 @@ +{ + "ALB": "Albania", + "AND": "Andorra", + "ANT": "Netherlands Antilles", + "ARG": "Argentina", + "ARM": "Armenia", + "ASM": "American Samoa", + "AUS": "Australia", + "AUT": "Austria", + "AZE": "Azerbaijan", + "BEL": "Belgium", + "BGR": "Bulgaria", + "BHR": "Bahrain", + "BHS": "Bahamas", + "BIH": "Bosnia and Herzegovina", + "BLR": "Belarus", + "BLZ": "Belize", + "BOL": "Bolivia", + "BRB": "Barbados", + "BRN": "Brunei", + "CAF": "Central African Republic", + "CAN": "Canada", + "CHE": "Switzerland", + "CHL": "Chile", + "COG": "Congo", + "COL": "Colombia", + "CRI": "Costa Rica", + "CUB": "Cuba", + "CYP": "Cyprus", + "CZE": "Czechia", + "DEU": "Germany", + "DNK": "Denmark", + "DOM": "Dominican Republic", + "DZA": "Algeria", + "ECU": "Ecuador", + "EGY": "Egypt", + "ESP": "Spain", + "EST": "Estonia", + "FIN": "Finland", + "FJI": "Fiji", + "FRA": "France", + "FRO": "Faroe Islands", + "GEO": "Georgia", + "GIN": "Guinea", + "GLP": "Guadeloupe", + "GRC": "Greece", + "GRL": "Greenland", + "GTM": "Guatemala", + "GUF": "French Guiana", + "GUM": "Guam", + "GUY": "Guyana", + "HND": "Honduras", + "HRV": "Croatia", + "HUN": "Hungary", + "IRL": "Ireland", + "IRQ": "Iraq", + "ISL": "Iceland", + "ISR": "Israel", + "ITA": "Italy", + "JAM": "Jamaica", + "JOR": "Jordan", + "JPN": "Japan", + "KAZ": "Kazakhstan", + "KGZ": "Kyrgyzstan", + "KOR": "South Korea", + "KWT": "Kuwait", + "LBY": "Libya", + "LTU": "Lithuania", + "LUX": "Luxembourg", + "LVA": "Latvia", + "MAC": "Macao", + "MAR": "Morocco", + "MCO": "Monaco", + "MDA": "Moldova", + "MDV": "Maldives", + "MEX": "Mexico", + "MKD": "North Macedonia", + "MLT": "Malta", + "MNG": "Mongolia", + "MTQ": "Martinique", + "MUS": "Mauritius", + "MYS": "Malaysia", + "MYT": "Mayotte", + "NAM": "Namibia", + "NCL": "New Caledonia", + "NIC": "Nicaragua", + "NLD": "Netherlands", + "NOR": "Norway", + "NRU": "Nauru", + "NZL": "New Zealand", + "OMN": "Oman", + "PAK": "Pakistan", + "PAN": "Panama", + "PER": "Peru", + "PHL": "Philippines", + "PLW": "Palau", + "PNG": "Papua New Guinea", + "POL": "Poland", + "PRI": "Puerto Rico", + "PRT": "Portugal", + "PYF": "French Polynesia", + "QAT": "Qatar", + "REU": "Reunion", + "ROM": "Romania", + "RUS": "Russia", + "SGP": "Singapore", + "SHN": "Saint Helena", + "SLV": "El Salvador", + "SMR": "San Marino", + "SPM": "Saint Pierre and Miquelon", + "SRB": "Serbia", + "SVK": "Slovakia", + "SVN": "Slovenia", + "SWE": "Sweden", + "SYC": "Seychelles", + "TJK": "Tajikistan", + "TTO": "Trinidad and Tobago", + "TUN": "Tunisia", + "TUR": "Turkey", + "TWN": "Taiwan", + "UKR": "Ukraine", + "URY": "Uruguay", + "USA": "United States", + "UZB": "Uzbekistan", + "VEN": "Venezuela", + "VIR": "United States Virgin Islands", + "WSM": "Samoa", + "ZAF": "South Africa", + "GBREW": "England and Wales", + "GBRNI": "Northern Ireland", + "GBRSC": "Scotland", + "MON": "Yugoslavia" +} \ No newline at end of file diff --git a/etl/steps/data/garden/demography/2024-12-05/paternal_ages.meta.yml b/etl/steps/data/garden/demography/2024-12-05/paternal_ages.meta.yml new file mode 100644 index 00000000000..b975222933c --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-05/paternal_ages.meta.yml @@ -0,0 +1,1185 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + others: + note_on_average: |- + Kai and Klüsener provide data from various sources. Whenever there were multiple data available for a given year, we averaged them. + sources_counts: |- + Australia: + 1936-2020: UN DYB + + Austria: + 1958-1969: UN DYB + 1970-1982: National Statistic Bureau + 1983-1984: National Statistic Bureau; UN DYB (averaged) + 1985-1989: National Statistic Bureau + 1990-1998: National Statistic Bureau; UN DYB (averaged) + 1999-2006: National Statistic Bureau + 2007: National Statistic Bureau; UN DYB (averaged) + 2008: National Statistic Bureau + 2009-2013: National Statistic Bureau; UN DYB (averaged) + 2014: National Statistic Bureau + 2015-2018: National Statistic Bureau; UN DYB (averaged) + 2019: National Statistic Bureau + + Belarus: + 1994-2018: UN DYB + + Belgium: + 1939-2014: UN DYB + + Bulgaria: + 1949-2021: UN DYB + + Canada: + 1944-2020: UN DYB + + Chile: + 1992-2020: UN DYB + + Croatia: + 2007-2020: UN DYB + + Czechia: + 1956-2021: UN DYB + + Denmark: + 1957-1985: UN DYB + 1986-1989: National Statistic Bureau + 1990: National Statistic Bureau; UN DYB (averaged) + 1991: National Statistic Bureau + 1992-1991: National Statistic Bureau; UN DYB (averaged) + 1992-2006: National Statistic Bureau + 2007: National Statistic Bureau; UN DYB (averaged) + 2008: National Statistic Bureau + 2009-2008: National Statistic Bureau; UN DYB (averaged) + 2009: National Statistic Bureau + 2010-2009: National Statistic Bureau; UN DYB (averaged) + 2010: National Statistic Bureau + 2011-2010: National Statistic Bureau; UN DYB (averaged) + 2011: National Statistic Bureau + 2012-2011: National Statistic Bureau; UN DYB (averaged) + 2012: National Statistic Bureau + 2013: National Statistic Bureau; UN DYB (averaged) + 2014: National Statistic Bureau + 2015-2014: National Statistic Bureau; UN DYB (averaged) + 2015: National Statistic Bureau + 2016-2018: National Statistic Bureau; UN DYB (averaged) + 2019: National Statistic Bureau + 2020: National Statistic Bureau; UN DYB (averaged) + + England and Wales: + 1961-1985: UN DYB + + Estonia: + 1989: National Statistic Bureau + 1990: National Statistic Bureau; UN DYB (averaged) + 1991-1990: National Statistic Bureau + 1991-1997: National Statistic Bureau; UN DYB (averaged) + 1998-2006: National Statistic Bureau + 2007-2006: National Statistic Bureau; UN DYB (averaged) + 2007: National Statistic Bureau + 2008-2007: National Statistic Bureau; UN DYB (averaged) + 2008: National Statistic Bureau + 2009-2008: National Statistic Bureau; UN DYB (averaged) + 2009: National Statistic Bureau + 2010: National Statistic Bureau; UN DYB (averaged) + 2011: National Statistic Bureau + 2012-2011: National Statistic Bureau; UN DYB (averaged) + 2012: National Statistic Bureau + 2013-2012: National Statistic Bureau; UN DYB (averaged) + 2013: National Statistic Bureau + 2014-2013: National Statistic Bureau; UN DYB (averaged) + 2014: National Statistic Bureau + 2015-2014: National Statistic Bureau; UN DYB (averaged) + 2015: National Statistic Bureau + 2016-2019: UN DYB + + Finland: + 1939-1986: UN DYB + 1987-1989: National Statistic Bureau + 1990-1989: National Statistic Bureau; UN DYB (averaged) + 1990-1993: National Statistic Bureau + 1994: National Statistic Bureau; UN DYB (averaged) + 1995-1994: National Statistic Bureau + 1995: National Statistic Bureau; UN DYB (averaged) + 1996-1995: National Statistic Bureau + 1996: National Statistic Bureau; UN DYB (averaged) + 1997: National Statistic Bureau + 1998: National Statistic Bureau; UN DYB (averaged) + 1999-2006: National Statistic Bureau + 2007-2006: National Statistic Bureau; UN DYB (averaged) + 2007: National Statistic Bureau + 2008-2007: National Statistic Bureau; UN DYB (averaged) + 2008: National Statistic Bureau + 2009-2008: National Statistic Bureau; UN DYB (averaged) + 2009: National Statistic Bureau + 2010: National Statistic Bureau; UN DYB (averaged) + 2011: National Statistic Bureau + 2012-2011: National Statistic Bureau; UN DYB (averaged) + 2012-2013: National Statistic Bureau + 2014-2013: National Statistic Bureau; UN DYB (averaged) + 2014: National Statistic Bureau + 2015-2014: National Statistic Bureau; UN DYB (averaged) + 2015: National Statistic Bureau + 2016-2019: UN DYB + + France: + 1936-2005: UN DYB + 2006: National Statistic Bureau + 2007: National Statistic Bureau; UN DYB (averaged) + 2008: National Statistic Bureau + 2009-2008: National Statistic Bureau; UN DYB (averaged) + 2009: National Statistic Bureau + 2010-2009: National Statistic Bureau; UN DYB (averaged) + 2010-2011: National Statistic Bureau + 2012: National Statistic Bureau; UN DYB (averaged) + 2013: National Statistic Bureau + 2014-2013: National Statistic Bureau; UN DYB (averaged) + 2014: National Statistic Bureau + 2015-2014: National Statistic Bureau; UN DYB (averaged) + 2015: National Statistic Bureau + 2016-2020: UN DYB + + Germany: + 1991-1999: UN DYB + 2000-2006: National Statistic Bureau + 2007-2006: National Statistic Bureau; UN DYB (averaged) + 2007-2010: National Statistic Bureau + 2011-2010: National Statistic Bureau; UN DYB (averaged) + 2011: National Statistic Bureau + 2012-2011: National Statistic Bureau; UN DYB (averaged) + 2012: National Statistic Bureau + 2013-2012: National Statistic Bureau; UN DYB (averaged) + 2013: National Statistic Bureau + 2014: National Statistic Bureau; UN DYB (averaged) + 2015-2017: National Statistic Bureau + 2018-2017: National Statistic Bureau; UN DYB (averaged) + 2018: National Statistic Bureau + 2019-2018: National Statistic Bureau; UN DYB (averaged) + 2019-2020: National Statistic Bureau + + Greece: + 1981-2019: UN DYB + + Hungary: + 1950-2020: UN DYB + + Iceland: + 1951-1980: UN DYB + 1981-1980: Statistics Iceland; UN DYB (averaged) + 1981: UN DYB + 1982-1981: Statistics Iceland; UN DYB (averaged) + 1982: UN DYB + 1983-1982: Statistics Iceland; UN DYB (averaged) + 1983: UN DYB + 1984-1983: Statistics Iceland; UN DYB (averaged) + 1984: UN DYB + 1985-1989: Statistics Iceland + 1990-1989: Statistics Iceland; UN DYB (averaged) + 1990: UN DYB + 1991-1990: Statistics Iceland; UN DYB (averaged) + 1991: UN DYB + 1992-1991: Statistics Iceland; UN DYB (averaged) + 1992: UN DYB + 1993-1992: Statistics Iceland; UN DYB (averaged) + 1993: UN DYB + 1994: Statistics Iceland + 1995-1994: Statistics Iceland; UN DYB (averaged) + 1995: UN DYB + 1996: Statistics Iceland; UN DYB (averaged) + 1997: UN DYB + 1998-2006: Statistics Iceland + 2007-2015: Statistics Iceland; UN DYB (averaged) + 2016: Statistics Iceland + 2017-2019: Statistics Iceland; UN DYB (averaged) + 2020: Statistics Iceland + 2021: UN DYB + + Ireland: + 2006: UN DYB + + Israel: + 1983-2016: UN DYB + + Italy: + 1958-2019: UN DYB + + Japan: + 1952-2021: UN DYB + + Latvia: + 1990-1999: UN DYB + 2000-2006: National Statistic Bureau + 2007-2010: National Statistic Bureau; UN DYB (averaged) + 2011: National Statistic Bureau + 2012-2014: National Statistic Bureau; UN DYB (averaged) + 2015-2019: UN DYB + + Lithuania: + 1990-2020: UN DYB + + Luxembourg: + 1960-2021: UN DYB + + Netherlands: + 1937-1995: UN DYB + 1996-1995: National Statistic Bureau + 1996: UN DYB + 1997-2009: National Statistic Bureau + 2010: National Statistic Bureau; UN DYB (averaged) + 2011-2013: National Statistic Bureau + 2014: National Statistic Bureau; UN DYB (averaged) + 2015-2019: National Statistic Bureau + + New Zealand: + 1948-2020: UN DYB + + Northern Ireland: + 1974-1985: UN DYB + + Norway: + 1953-2021: UN DYB + + Poland: + 1958-2019: UN DYB + + Portugal: + 1980-2021: UN DYB + + Russia: + 1990-2012: UN DYB + + Scotland: + 1961-1985: UN DYB + + Slovakia: + 1956-2019: UN DYB + + Slovenia: + 1990-2008: UN DYB + 2009-2014: National Statistic Bureau; UN DYB (averaged) + 2015-2019: UN DYB + + South Korea: + 2006-2020: UN DYB + + Spain: + 1936-1995: UN DYB + 1996-1997: National Statistic Bureau; UN DYB (averaged) + 1998-2006: National Statistic Bureau + 2007-2006: National Statistic Bureau; UN DYB (averaged) + 2007-2008: National Statistic Bureau + 2009-2008: National Statistic Bureau; UN DYB (averaged) + 2009: National Statistic Bureau + 2010-2009: National Statistic Bureau; UN DYB (averaged) + 2010: National Statistic Bureau + 2011-2010: National Statistic Bureau; UN DYB (averaged) + 2011: National Statistic Bureau + 2012: National Statistic Bureau; UN DYB (averaged) + 2013-2015: National Statistic Bureau + 2016-2015: UN DYB + 2016: National Statistic Bureau + 2017-2016: UN DYB + 2017: National Statistic Bureau + 2018-2017: UN DYB + 2018: National Statistic Bureau + 2019-2018: UN DYB + 2019: National Statistic Bureau + 2020-2019: UN DYB + 2020: National Statistic Bureau + + Sweden: + 1901-2015: National Statistic Bureau + + Switzerland: + 1979-1978: National Statistic Bureau + 1979: National Statistic Bureau; UN DYB (averaged) + 1980-1979: National Statistic Bureau + 1980: National Statistic Bureau; UN DYB (averaged) + 1981-1980: National Statistic Bureau + 1981: National Statistic Bureau; UN DYB (averaged) + 1982-1981: National Statistic Bureau + 1982: National Statistic Bureau; UN DYB (averaged) + 1983-1982: National Statistic Bureau + 1983: National Statistic Bureau; UN DYB (averaged) + 1984: National Statistic Bureau + 1985: National Statistic Bureau; UN DYB (averaged) + 1986-1989: National Statistic Bureau + 1990: National Statistic Bureau; UN DYB (averaged) + 1991-1990: National Statistic Bureau + 1991: National Statistic Bureau; UN DYB (averaged) + 1992-1991: National Statistic Bureau + 1992: National Statistic Bureau; UN DYB (averaged) + 1993: National Statistic Bureau + 1994-1993: National Statistic Bureau; UN DYB (averaged) + 1994: National Statistic Bureau + 1995: National Statistic Bureau; UN DYB (averaged) + 1996-1995: National Statistic Bureau + 1996: National Statistic Bureau; UN DYB (averaged) + 1997: National Statistic Bureau + 1998-1997: National Statistic Bureau; UN DYB (averaged) + 1998-2005: National Statistic Bureau + 2006: National Statistic Bureau; UN DYB (averaged) + 2007: National Statistic Bureau + 2008: National Statistic Bureau; UN DYB (averaged) + 2009: National Statistic Bureau + 2010: National Statistic Bureau; UN DYB (averaged) + 2011-2010: National Statistic Bureau + 2011: National Statistic Bureau; UN DYB (averaged) + 2012-2011: National Statistic Bureau + 2012: National Statistic Bureau; UN DYB (averaged) + 2013: National Statistic Bureau + 2014-2013: National Statistic Bureau; UN DYB (averaged) + 2014: National Statistic Bureau + 2015: UN DYB + 2016-2015: National Statistic Bureau + 2016: UN DYB + 2017-2016: National Statistic Bureau + 2017: UN DYB + 2018-2017: National Statistic Bureau + 2018: UN DYB + 2019: National Statistic Bureau; UN DYB (averaged) + 2020: UN DYB + + Ukraine: + 1995-2013: UN DYB + + United States: + 1933-1935: US Census Bureau - all births + 1936-1945: US Census Bureau - all births; UN DYB (averaged) + 1946-1949: UN DYB + 1950: US Census Bureau - all births; UN DYB (averaged) + 1951-1959: UN DYB + 1960: US Census Bureau - all births; UN DYB (averaged) + 1961-2019: UN DYB + sources_rates: |- + Albania: + 1955-2021: UN DYB + + Algeria: + 1948-1955: UN DYB + + American Samoa: + 1956-1960: UN DYB + + Argentina: + 1954-1961: UN DYB + + Armenia: + 2018-2020: UN DYB + + Australia: + 1936-1974: UN DYB + 1975: HFC + 1976-1975: HFC; UN DYB (averaged) + 1976: UN DYB + 1977-1980: HFC; UN DYB (averaged) + 1981: UN DYB + 1982-1983: HFC; UN DYB (averaged) + 1984-1989: HFC + 1990: HFC; UN DYB (averaged) + 1991: HFC + 1992-1995: HFC; UN DYB (averaged) + 1996-2006: HFC + 2007-2006: UN DYB + 2007: HFC + 2008: UN DYB + 2009-2008: HFC + 2009: UN DYB + 2010-2009: HFC + 2010: UN DYB + 2011-2010: HFC + 2011: UN DYB + 2012-2014: HFC; UN DYB (averaged) + 2015-2021: UN DYB + + Austria: + 1965-2018: UN DYB + + Azerbaijan: + 2004-2019: UN DYB + + Bahamas: + 1990-2007: UN DYB + + Bahrain: + 1990-2019: UN DYB + + Belarus: + 2007-2019: UN DYB + + Belgium: + 1940: Lognard (2010) + 1941: Statistical Yearbook of Belgium; Lognard (2010) (averaged) + 1942-1944: Lognard (2010) + 1945: Statistical Yearbook of Belgium; Lognard (2010) (averaged) + 1946-1949: Lognard (2010) + 1950: Statistical Yearbook of Belgium; Lognard (2010) (averaged) + 1951-1954: Lognard (2010) + 1955: Statistical Yearbook of Belgium; Lognard (2010) (averaged) + 1956-1957: Lognard (2010) + 1958-1973: Statistical Yearbook of Belgium; Lognard (2010) (averaged) + 1974-1995: Lognard (2010) + + Bolivia: + 1951: UN DYB + + Bosnia and Herzegovina: + 1960-2010: UN DYB + + Brunei: + 1971-2021: UN DYB + + Bulgaria: + 1956-2021: UN DYB + + Canada: + 1944-1973: UN DYB + 1974-1975: HFC + 1976-1979: HFC; UN DYB (averaged) + 1980-1989: HFC + 1990: HFC; UN DYB (averaged) + 1991: HFC + 1992-1997: HFC; UN DYB (averaged) + 1998-2004: HFC + 2005: HFC; UN DYB (averaged) + 2006: HFC + 2007-2009: HFC; UN DYB (averaged) + 2010-2016: HFC + 2017-2021: UN DYB + + Central African Republic: + 1960: UN DYB + + Chile: + 1952-2020: UN DYB + + Colombia: + 1951-2022: UN DYB + + Congo: + 1961: UN DYB + + Costa Rica: + 1955-2022: UN DYB + + Croatia: + 1960-2021: UN DYB + + Cuba: + 1970-2021: UN DYB + + Cyprus: + 1948-2016: UN DYB + + Czechia: + 1961-2021: UN DYB + + Denmark: + 1957-1985: UN DYB + 1986-1989: HFC + 1990: UN DYB + 1991-1997: HFC; UN DYB (averaged) + 1998-2006: HFC + 2007-2015: HFC; UN DYB (averaged) + 2016-2020: UN DYB + + Dominican Republic: + 1960-2007: UN DYB + + Egypt: + 1976-2012: UN DYB + + El Salvador: + 1951-2014: UN DYB + + England and Wales: + 1961-1963: UN DYB + 1964-1963: UN DYB; NSSEC - Institute for Social and Economic Research (averaged) + 1964: UN DYB + 1965-1967: UN DYB; NSSEC - Institute for Social and Economic Research (averaged) + 1968: UN DYB + 1969-1972: UN DYB; NSSEC - Institute for Social and Economic Research (averaged) + 1973-1972: UN DYB + 1973: UN DYB; NSSEC - Institute for Social and Economic Research (averaged) + 1974: NSSEC - Institute for Social and Economic Research + 1975-1978: UN DYB; NSSEC - Institute for Social and Economic Research (averaged) + 1979: UN DYB + 1980-1984: UN DYB; NSSEC - Institute for Social and Economic Research (averaged) + 1985: UN DYB + 1986-2020: NSSEC - Institute for Social and Economic Research + + Estonia: + 1989-1992: HFC + 1993-1997: HFC; UN DYB (averaged) + 1998-2006: HFC + 2007-2014: HFC; UN DYB (averaged) + 2015-2020: UN DYB + + Faroe Islands: + 2015-2022: UN DYB + + Fiji: + 1953-2004: UN DYB + + Finland: + 1939-1938: National Statistic Bureau + 1939: UN DYB + 1940-1939: National Statistic Bureau + 1940: UN DYB + 1941-1940: National Statistic Bureau + 1941: UN DYB + 1942-1941: National Statistic Bureau + 1942: UN DYB + 1943: National Statistic Bureau + 1944: UN DYB + 1945-1944: National Statistic Bureau + 1945: UN DYB + 1946-1945: National Statistic Bureau + 1946: UN DYB + 1947: National Statistic Bureau + 1948-1947: UN DYB + 1948: National Statistic Bureau + 1949: UN DYB + 1950: National Statistic Bureau + 1951: UN DYB + 1952-1951: National Statistic Bureau + 1952-1954: UN DYB + 1955: National Statistic Bureau + 1956: UN DYB + 1957-1956: National Statistic Bureau + 1957: UN DYB + 1958-1957: National Statistic Bureau + 1958-1986: UN DYB + 1987-1993: HFC + 1994-1993: UN DYB + 1994: National Statistic Bureau; HFC (averaged) + 1995: UN DYB + 1996-1995: National Statistic Bureau; HFC (averaged) + 1996: UN DYB + 1997-1996: National Statistic Bureau; HFC (averaged) + 1997: UN DYB + 1998: National Statistic Bureau; HFC (averaged) + 1999-2006: HFC + 2007-2012: HFC; UN DYB (averaged) + 2013: HFC + 2014-2015: HFC; UN DYB (averaged) + 2016-2019: UN DYB + + France: + 1900: Brouard; N. (1977) doi:10.2307/1531392 (averaged) + 1901-1900: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1901: Insee; Etat civil (averaged) + 1902-1901: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1902: Insee; Etat civil (averaged) + 1903-1902: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1903: Insee; Etat civil (averaged) + 1904-1903: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1904: Insee; Etat civil (averaged) + 1905-1904: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1905: Insee; Etat civil (averaged) + 1906-1905: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1906: Insee; Etat civil (averaged) + 1907: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1908: Insee; Etat civil (averaged) + 1909-1908: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1909: Insee; Etat civil (averaged) + 1910-1909: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1910: Insee; Etat civil (averaged) + 1911-1910: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1911: Insee; Etat civil (averaged) + 1912-1911: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1912: Insee; Etat civil (averaged) + 1913-1912: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1913: Insee; Etat civil (averaged) + 1914: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1915: Insee; Etat civil (averaged) + 1916-1915: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1916: Insee; Etat civil (averaged) + 1917-1916: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1917: Insee; Etat civil (averaged) + 1918-1917: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1918: Insee; Etat civil (averaged) + 1919-1918: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1919: Insee; Etat civil (averaged) + 1920-1919: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1920: Insee; Etat civil (averaged) + 1921: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1922: Insee; Etat civil (averaged) + 1923-1922: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1923: Insee; Etat civil (averaged) + 1924-1923: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1924: Insee; Etat civil (averaged) + 1925-1924: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1925: Insee; Etat civil (averaged) + 1926-1925: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1926: Insee; Etat civil (averaged) + 1927: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1928-1927: Insee; Etat civil (averaged) + 1928: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1929: Insee; Etat civil (averaged) + 1930-1929: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1930: Insee; Etat civil (averaged) + 1931-1930: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1931: Insee; Etat civil (averaged) + 1932-1931: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1932: Insee; Etat civil (averaged) + 1933-1932: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1933: Insee; Etat civil (averaged) + 1934-1933: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1934: Insee; Etat civil (averaged) + 1935: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1936: Insee; Etat civil; UN DYB (averaged) + 1937-1936: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1937: Insee; Etat civil; UN DYB (averaged) + 1938-1937: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1938: Insee; Etat civil; UN DYB (averaged) + 1939-1938: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1939: Insee; Etat civil; UN DYB (averaged) + 1940-1939: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1940: Insee; Etat civil; UN DYB (averaged) + 1941: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1942-1941: Insee; Etat civil; UN DYB (averaged) + 1942: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1943: Insee; Etat civil; UN DYB (averaged) + 1944-1943: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1944: Insee; Etat civil; UN DYB (averaged) + 1945-1944: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1945: Insee; Etat civil; UN DYB (averaged) + 1946-1945: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1946: Insee; Etat civil; UN DYB (averaged) + 1947-1946: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1947: Insee; Etat civil (averaged) + 1948-1947: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1948: Insee; Etat civil; UN DYB (averaged) + 1949: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1950: Insee; Etat civil; UN DYB (averaged) + 1951-1950: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1951: Insee; Etat civil (averaged) + 1952-1951: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1952: Insee; Etat civil (averaged) + 1953-1952: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1953: Insee; Etat civil (averaged) + 1954-1953: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1954: Insee; Etat civil (averaged) + 1955-1954: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1955: Insee; Etat civil (averaged) + 1956: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1957: Insee; Etat civil (averaged) + 1958-1957: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1958: Insee; Etat civil (averaged) + 1959-1958: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1959: Insee; Etat civil (averaged) + 1960-1959: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1960: Insee; Etat civil (averaged) + 1961-1960: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1961: Insee; Etat civil (averaged) + 1962-1961: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1962: Insee; Etat civil (averaged) + 1963-1962: Insee; Etat civil; UN DYB (averaged) + 1963: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1964: Insee; Etat civil (averaged) + 1965-1964: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1965: Insee; Etat civil (averaged) + 1966-1965: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1966: Insee; Etat civil (averaged) + 1967-1966: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1967: Insee; Etat civil; UN DYB (averaged) + 1968-1967: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1968: Insee; Etat civil (averaged) + 1969-1968: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1969: Insee; Etat civil (averaged) + 1970: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1971: Insee; Etat civil (averaged) + 1972-1971: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1972: Insee; Etat civil; UN DYB (averaged) + 1973-1972: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1973: Insee; Etat civil (averaged) + 1974-1973: Brouard; N. (1977) doi:10.2307/1531392; Insee; Etat civil (averaged) + 1974: Insee; Etat civil (averaged) + 1975-1974: Insee; Etat civil; UN DYB (averaged) + 1975: Insee; Etat civil (averaged) + 1976-1975: Insee; Etat civil; UN DYB (averaged) + 1976: Insee; Etat civil (averaged) + 1977: Insee; Etat civil; UN DYB (averaged) + 1978-1979: Insee; Etat civil (averaged) + 1980: Insee; Etat civil; UN DYB (averaged) + 1981-1990: Insee; Etat civil (averaged) + 1991-1990: Insee; Etat civil; UN DYB (averaged) + 1991-1997: Insee; Etat civil (averaged) + 1998: Insee; Etat civil; HFC (averaged) + 1999: Insee; Etat civil (averaged) + 2000-1999: Insee; Etat civil; HFC (averaged) + 2000: Insee; Etat civil (averaged) + 2001-2000: Insee; Etat civil; HFC (averaged) + 2001: Insee; Etat civil (averaged) + 2002-2001: Insee; Etat civil; HFC (averaged) + 2002: Insee; Etat civil (averaged) + 2003-2002: Insee; Etat civil; HFC (averaged) + 2003: Insee; Etat civil (averaged) + 2004: Insee; Etat civil; HFC (averaged) + 2005-2004: Insee; Etat civil (averaged) + 2005-2006: Insee; Etat civil; HFC (averaged) + 2007-2010: Insee; Etat civil; HFC; UN DYB (averaged) + 2011: Insee; Etat civil; HFC (averaged) + 2012-2013: Insee; Etat civil; HFC; UN DYB (averaged) + 2014-2017: Insee; Etat civil; UN DYB (averaged) + 2018: Insee; Etat civil (averaged) + 2019: Insee; Etat civil; UN DYB (averaged) + 2020: UN DYB + + French Guiana: + 1961-2021: UN DYB + + French Polynesia: + 2018-2021: UN DYB + + Georgia: + 2016: UN DYB + + Germany: + 1950-1990: UN DYB + 1991-1990: National Statistic Bureau; HFC (averaged) + 1991: UN DYB + 1992-1991: National Statistic Bureau; HFC (averaged) + 1992: UN DYB + 1993-1992: National Statistic Bureau; HFC (averaged) + 1993: UN DYB + 1994-1993: National Statistic Bureau; HFC (averaged) + 1994: UN DYB + 1995-2006: National Statistic Bureau; HFC (averaged) + 2007: National Statistic Bureau; HFC; UN DYB (averaged) + 2008-2010: National Statistic Bureau; HFC (averaged) + 2011-2013: National Statistic Bureau; HFC; UN DYB (averaged) + 2014-2015: National Statistic Bureau; UN DYB (averaged) + 2016-2017: National Statistic Bureau + 2018-2019: National Statistic Bureau; UN DYB (averaged) + 2020: National Statistic Bureau + + Greece: + 1961-2020: UN DYB + + Greenland: + 1993-2006: UN DYB + + Guadeloupe: + 1954-2021: UN DYB + + Guam: + 1950-1980: UN DYB + + Guatemala: + 1950-2020: UN DYB + + Guinea: + 1955: UN DYB + + Guyana: + 1954-1970: UN DYB + + Honduras: + 1951-1981: UN DYB + + Hungary: + 1950-1969: UN DYB + 1970-1973: HFC; UN DYB (averaged) + 1974: UN DYB + 1975-1979: HFC; UN DYB (averaged) + 1980: UN DYB + 1981-1984: HFC; UN DYB (averaged) + 1985: UN DYB + 1986-1989: HFC + 1990-1989: HFC; UN DYB (averaged) + 1990: UN DYB + 1991: HFC; UN DYB (averaged) + 1992: UN DYB + 1993: HFC + 1994-1993: HFC; UN DYB (averaged) + 1994: UN DYB + 1995-1994: HFC; UN DYB (averaged) + 1995: UN DYB + 1996: HFC; UN DYB (averaged) + 1997: UN DYB + 1998-1997: HFC; UN DYB (averaged) + 1998: UN DYB + 1999-2006: HFC + 2007-2014: HFC; UN DYB (averaged) + 2015-2021: UN DYB + + Iceland: + 1951-1954: UN DYB + 1955-1966: Statistics Iceland; UN DYB (averaged) + 1967: UN DYB + 1968: Statistics Iceland; UN DYB (averaged) + 1969-1968: UN DYB + 1969-1979: Statistics Iceland; UN DYB (averaged) + 1980: UN DYB + 1981-1983: Statistics Iceland; UN DYB (averaged) + 1984: UN DYB + 1985-1989: Statistics Iceland + 1990-1993: Statistics Iceland; UN DYB (averaged) + 1994: Statistics Iceland + 1995-1997: Statistics Iceland; UN DYB (averaged) + 1998-2006: Statistics Iceland + 2007-2015: Statistics Iceland; UN DYB (averaged) + 2016: Statistics Iceland + 2017-2019: Statistics Iceland; UN DYB (averaged) + 2020: Statistics Iceland + + Iraq: + 1975-1977: UN DYB + + Ireland: + 2006: UN DYB + + Israel: + 1944-2021: UN DYB + + Italy: + 1967-1998: UN DYB + 1999-2005: HFC + 2006-2011: HFC; UN DYB (averaged) + 2012: HFC + 2013-2015: HFC; UN DYB (averaged) + 2016-2021: UN DYB + + Jamaica: + 1947-1956: UN DYB + + Japan: + 1960-2008: UN DYB + 2009-2008: HFC + 2009: UN DYB + 2010-2009: HFC + 2010: UN DYB + 2011: HFC + 2012: UN DYB + 2013-2012: HFC + 2013: UN DYB + 2014-2013: HFC + 2014-2021: UN DYB + + Jordan: + 1969-1979: UN DYB + + Kazakhstan: + 2006-2022: UN DYB + + Kuwait: + 1965-1984: UN DYB + + Kyrgyzstan: + 1996-2021: UN DYB + + Latvia: + 1995-2021: UN DYB + + Libya: + 1973: UN DYB + + Lithuania: + 1993-2021: UN DYB + + Luxembourg: + 1960-2021: UN DYB + + Macao: + 1960-2022: UN DYB + + Malaysia: + 2008-2021: UN DYB + + Maldives: + 2010-2019: UN DYB + + Malta: + 1958-2020: UN DYB + + Martinique: + 1954-2021: UN DYB + + Mauritius: + 1991-2022: UN DYB + + Mayotte: + 2021: UN DYB + + Mexico: + 1976-2020: UN DYB + + Moldova: + 2007-2020: UN DYB + + Monaco: + 1951: UN DYB + + Mongolia: + 2018-2022: UN DYB + + Morocco: + 1951: UN DYB + + Namibia: + 1936-1960: UN DYB + + Netherlands: + 1936-2014: UN DYB + + Netherlands Antilles: + 1971: UN DYB + + New Caledonia: + 2007-2015: UN DYB + + New Zealand: + 1936-2020: UN DYB + + Nicaragua: + 1950-1965: UN DYB + + North Macedonia: + 1960-2018: UN DYB + + Northern Ireland: + 1981: UN DYB + + Norway: + 1953-1998: UN DYB + 1999-2006: National Statistic Bureau + 2007-2012: National Statistic Bureau; UN DYB (averaged) + 2013-2020: National Statistic Bureau + 2021: UN DYB + + Oman: + 2017-2022: UN DYB + + Pakistan: + 1968: UN DYB + + Palau: + 1965-1979: UN DYB + + Panama: + 1949-2021: UN DYB + + Papua New Guinea: + 1947: UN DYB + + Peru: + 1960-1973: UN DYB + + Philippines: + 1952-2021: UN DYB + + Poland: + 1950-1985: UN DYB + 1986-1989: HFC + 1990-1989: UN DYB + 1990-1995: HFC; UN DYB (averaged) + 1996: UN DYB + 1997-1996: HFC; UN DYB (averaged) + 1997: UN DYB + 1998-2006: HFC + 2007: HFC; UN DYB (averaged) + 2008: HFC + 2009-2012: HFC; UN DYB (averaged) + 2013-2014: HFC + 2015-2021: UN DYB + + Portugal: + 1963-1979: UN DYB + 1980: HFC; UN DYB (averaged) + 1981: UN DYB + 1982: HFC + 1983-1985: HFC; UN DYB (averaged) + 1986-1990: HFC + 1991-1990: HFC; UN DYB (averaged) + 1991: UN DYB + 1992-1993: HFC; UN DYB (averaged) + 1994: HFC + 1995-1997: HFC; UN DYB (averaged) + 1998-2006: HFC + 2007-2009: HFC; UN DYB (averaged) + 2010: HFC + 2011-2015: HFC; UN DYB (averaged) + 2016-2021: UN DYB + + Puerto Rico: + 1959-2008: UN DYB + + Qatar: + 2007-2020: UN DYB + + Reunion: + 1954-2021: UN DYB + + Romania: + 1961-2021: UN DYB + + Russia: + 2011-2012: UN DYB + + Samoa: + 1976: UN DYB + + San Marino: + 1962-1985: UN DYB + + Scotland: + 1964-1985: UN DYB + + Serbia: + 1960-2021: UN DYB + + Seychelles: + 1971-1973: UN DYB + + Singapore: + 1984-2021: UN DYB + + Slovakia: + 1961-2021: UN DYB + + Slovenia: + 1960-2021: UN DYB + + South Africa: + 1943-2020: UN DYB + + South Korea: + 1981-2020: UN DYB + + Spain: + 1960-1974: UN DYB + 1975-1977: HFC + 1978-1977: HFC; UN DYB (averaged) + 1978: UN DYB + 1979-1990: HFC + 1991-1990: HFC; UN DYB (averaged) + 1991: UN DYB + 1992-1997: HFC; UN DYB (averaged) + 1998-2006: HFC + 2007: HFC; UN DYB (averaged) + 2008: HFC + 2009-2013: HFC; UN DYB (averaged) + 2014-2015: HFC + 2016-2021: UN DYB + + Sweden: + 1936-1967: UN DYB + 1968-1973: HFC + 1974: UN DYB + 1975-1979: HFC + 1980-1979: UN DYB + 1980-1984: HFC + 1985: UN DYB + 1986-1989: HFC + 1990: UN DYB + 1991-1990: HFC + 1991: UN DYB + 1992-1991: HFC + 1992: UN DYB + 1993: HFC + 1994: UN DYB + 1995: HFC + 1996-1995: UN DYB + 1996: HFC + 1997: UN DYB + 1998-2006: HFC + 2007-2012: HFC; UN DYB (averaged) + 2013-2015: HFC + + Switzerland: + 1980-2020: UN DYB + + Taiwan: + 1950-1997: UN DYB + 1998-2014: HFC + + Tajikistan: + 2020-2021: UN DYB + + Trinidad and Tobago: + 1954-1997: UN DYB + + Tunisia: + 1965-1980: UN DYB + + Turkey: + 2010-2019: UN DYB + + Ukraine: + 1995-2021: UN DYB + + United States: + 1936-1968: UN DYB + 1969-1984: HFC; UN DYB (averaged) + 1985-1990: HFC + 1991: HFC; UN DYB (averaged) + 1992: HFC + 1993-1997: HFC; UN DYB (averaged) + 1998-2007: HFC + 2008-2009: HFC; UN DYB (averaged) + 2010-2011: HFC + 2012: HFC; UN DYB (averaged) + 2013-2014: HFC + 2015: HFC; UN DYB (averaged) + + United States Virgin Islands: + 1960-1980: UN DYB + + Uruguay: + 1963-2021: UN DYB + + Uzbekistan: + 2014-2022: UN DYB + + Venezuela: + 1950-2013: UN DYB + + Yugoslavia: + 1960-2009: UN DYB + common: + unit: years + presentation: + topic_tags: + - Fertility Rate + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + update_period_days: 365 + +tables: + paternal_age: + variables: + mpac_fr: + title: Mean paternal age at childbirth (based on fertility rates) + description_short: |- + Average age of fathers at the time their children are born. Estimated using fertilty rates. + description_processing: |- + {definitions.others.note_on_average} + + ### Sources: + + {definitions.others.sources_counts} + mpac_mean: + title: Mean paternal age at childbirth (based on arithmetic mean) + description_short: |- + Average age of fathers at the time their children are born. Estimated using fertilty rates. + description_processing: |- + {definitions.others.note_on_average} + + ### Sources: + + {definitions.others.sources_counts} + mpac_rates: + title: Mean paternal age at childbirth (unknown method) + description_short: |- + Average age of fathers at the time their children are born. + description_processing: |- + {definitions.others.note_on_average} + + ### Sources: + + {definitions.others.sources_rates} diff --git a/etl/steps/data/garden/demography/2024-12-05/paternal_ages.py b/etl/steps/data/garden/demography/2024-12-05/paternal_ages.py new file mode 100644 index 00000000000..3c63bcc8314 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-05/paternal_ages.py @@ -0,0 +1,157 @@ +"""Load a meadow dataset and create a garden dataset. + +NOTES +This dataset relies on various sources. In this step, we consolidate them to have a single time-series. These are the distributions: + +tb_counts: +source % +UN DYB 71.823204 +National Statistic Bureau 25.839354 +Statistics Iceland 1.699958 +US Census Bureau - all births 0.637484 + +tb_rates: +source % +UN DYB 65.786933 +HFC 15.803025 +Insee, Etat civil 7.209527 +National Statistic Bureau 2.413904 +Brouard, N. (1977) doi:10.2307/1531392 2.413904 +Statistics Iceland 2.09205 +NSSEC - Institute for Social and Economic Research 1.834567 +Lognard (2010) 1.802382 +Statistical Yearbook of Belgium 0.643708 + +""" + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +COLUMNS_INDEX = ["country", "year", "source", "type"] +COLUMNS_COUNTS_METRICS = { + "mean_age_at_childbirth_based_demographic_rates": "mpac_fr", + "mean_age_at_childbirth_as_arithmetic_mean": "mpac_mean", +} +COLUMNS_RATES_METRICS = { + "meanage": "mpac_rates", +} + +TYPES_EXPECTED = [ + "marital births", + "all births", +] + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("paternal_ages") + + # Read table from meadow dataset. + tb_counts = ds_meadow.read("counts") + tb_rates = ds_meadow.read("rates") + + ## Keep relevant columns + tb_counts = clean_table( + tb_counts, + COLUMNS_COUNTS_METRICS, + ) + tb_rates = clean_table( + tb_rates, + COLUMNS_RATES_METRICS, + ) + + # Harmonize country names + tb_counts = geo.harmonize_countries( + df=tb_counts, + countries_file=paths.country_mapping_path, + warn_on_unused_countries=False, + ) + tb_rates = geo.harmonize_countries( + df=tb_rates, + countries_file=paths.country_mapping_path, + warn_on_unused_countries=False, + ) + + # Remove spurious categories + tb_counts = tb_counts.loc[tb_counts["type"].isin(TYPES_EXPECTED)] + tb_rates = tb_rates.loc[tb_rates["type"].isin(TYPES_EXPECTED)] + + # Average different sources + tb_counts = average_values_across_sources(tb_counts, ["mpac_fr", "mpac_mean"]) + tb_rates = average_values_across_sources(tb_rates, ["mpac_rates"]) + + # Combine tables + tb = merge_tables(tb_counts, tb_rates) + + # Process data. + tables = [ + tb.format(["country", "year", "type"]), + ] + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, + tables=tables, + check_variables_metadata=True, + default_metadata=ds_meadow.metadata, + ) + + # Save changes in the new garden dataset. + ds_garden.save() + + +def clean_table(tb, columns_metrics_rename): + # Drop column 'country' if exists + if "country" in tb.columns: + tb = tb.drop(columns="country") + + # Rename columns + columns_rename = {"code": "country", **columns_metrics_rename} + tb = tb.rename(columns=columns_rename) + + # Keep relevant columns + columns_keep = COLUMNS_INDEX + list(columns_metrics_rename.values()) + tb = tb.loc[:, columns_keep] + + # Drop NaNs + tb = tb.dropna(subset=columns_metrics_rename.values(), how="all") + + return tb + + +def average_values_across_sources(tb, columns): + tb = tb.groupby(["country", "year", "type"], as_index=False).agg( + { + **{col: "mean" for col in columns}, + "source": set, + } + ) + return tb + + +def merge_tables(tb_counts, tb_rates): + tb = tb_counts.merge( + tb_rates, + on=["country", "year", "type"], + how="outer", + suffixes=[ + "_counts", + "_rates", + ], + short_name="paternal_age", + ) + tb = tb[ + [col for col in COLUMNS_INDEX if col != "source"] + + list(COLUMNS_COUNTS_METRICS.values()) + + list(COLUMNS_RATES_METRICS.values()) + ] + return tb diff --git a/etl/steps/data/garden/demography/2024-12-06/shared.py b/etl/steps/data/garden/demography/2024-12-06/shared.py new file mode 100644 index 00000000000..e3ec4505cde --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-06/shared.py @@ -0,0 +1,171 @@ +import owid.catalog.processing as pr + +from etl.data_helpers import geo + +# Columns index +COLUMNS_INDEX = [ + "country", + "year", + "scenario", + "sex", + "age", + "education", +] + + +def make_table( + tb, + country_mapping_path, + dtypes=None, + all_single=False, + all_range=False, + cols_single=None, + cols_range=None, + per_10=None, + per_100=None, + per_1000=None, + div_10=None, + div_100=None, + div_1000=None, +): + dtypes = {**{"scenario": "UInt8", "country": "category"}, **(dtypes or {})} + tb = tb.astype(dtypes) + + if all_single: + tb["year"] = tb["year"].astype("Int32") + elif all_range: + assert tb["year"].str.contains("-").all(), "Some years are not ranges!" + tb["year"] = tb["year"].str.extract(r"(\d{4}\.?0?)$").astype("Float32").astype("Int32") + else: + tb = consolidate_year_single_and_ranges( + tb=tb, + cols_single=cols_single, + cols_range=cols_range, + ) + + # Ensure expected scenario IDs + assert set(tb["scenario"].unique()) == set(range(1, 6)) + + # Harmonize country names + tb = geo.harmonize_countries( + df=tb, + countries_file=country_mapping_path, + show_full_warning=False, + ) + + # Scale + tb = scale_values( + tb, per_10=per_10, per_100=per_100, per_1000=per_1000, div_10=div_10, div_100=div_100, div_1000=div_1000 + ) + return tb + + +def scale_values(tb, per_10=None, per_100=None, per_1000=None, div_10=None, div_100=None, div_1000=None): + if per_10 is not None: + for col in per_10: + tb[col] *= 10 + if per_100 is not None: + for col in per_100: + tb[col] *= 100 + if per_1000 is not None: + for col in per_1000: + tb[col] *= 1000 + if div_10 is not None: + for col in div_10: + tb[col] /= 10 + if div_100 is not None: + for col in div_100: + tb[col] /= 100 + if div_1000 is not None: + for col in div_1000: + tb[col] /= 1000 + return tb + + +def consolidate_year_single_and_ranges(tb, cols_single, cols_range): + # 1) Periods like "2020-2025" are mapped to 2025 + # 2) While doing 1, we should make sure that the tables are properly aligned + flag = tb["year"].str.contains("-") + + # Check columns for single year data + single_year_cols = set(tb.loc[~flag].dropna(axis=1, how="all").columns) - set(COLUMNS_INDEX) + assert single_year_cols == set(cols_single), f"Unexpected columns in single year data: {single_year_cols}" + + # Check columns for range year data + range_year_cols = set(tb.loc[flag].dropna(axis=1, how="all").columns) - set(COLUMNS_INDEX) + assert range_year_cols == set(cols_range), f"Unexpected columns in range year data: {range_year_cols}" + + # Fix year type + tb["year"] = tb["year"].str.extract(r"(\d{4}\.?0?)$").astype("Float32").astype("Int32") + + # Create two tables: year range and single year + cols_index = list(tb.columns.intersection(COLUMNS_INDEX)) + tb_single = tb[cols_index + cols_single].dropna(subset=cols_single, how="all") + tb_range = tb[cols_index + cols_range].dropna(subset=cols_range, how="all") + + # Merge back + tb = tb_single.merge(tb_range, on=cols_index, how="outer") + + return tb + + +def add_dim_some_education(tb): + """Add dimension "some education" to sex+age+education table. + + It only adds it for sex=total and age=total. + """ + SOME_EDUCATION = "some_education" + # Add education="some_education" (only for sex=total and age=total, and indicator 'pop') + cols_index = ["country", "year", "age", "sex", "scenario"] + tb_tmp = tb.loc[tb["education"].isin(["total", "no_education"]), cols_index + ["education", "pop"]] + tb_tmp = tb_tmp.pivot(index=cols_index, columns="education", values="pop").reset_index().dropna() + tb_tmp["some_education"] = tb_tmp["total"] - tb_tmp["no_education"] + assert (tb_tmp["some_education"] >= 0).all() + tb_tmp = tb_tmp.melt(id_vars=cols_index, value_vars=SOME_EDUCATION, var_name="education", value_name="pop") + + # Add new education + tb["education"] = tb["education"].cat.add_categories([SOME_EDUCATION]) + + dtypes = tb.dtypes + tb = pr.concat([tb, tb_tmp], ignore_index=True) + tb = tb.astype(dtypes) + + return tb + + +def add_dim_15plus(tb): + # Pivot table to have two columns: "0-14" and "total" + tb_adults = tb.loc[tb["age"].isin(["0-4", "5-9", "10-14", "total"]) & (tb["education"] != "total")] + cols_index = ["country", "scenario", "sex", "education", "year"] + tb_adults = tb_adults.pivot(index=cols_index, columns="age", values="pop").reset_index() + # Only estimate values for adults when "total" is not NA + tb_adults = tb_adults.dropna(subset=["total"]) + # Estimate adults as "0-14" - 15+ + # Fill with zero NAs of agr group "0-14". NAs mostly come from 'doesn't apply' (e.g. primary education for 0-14) + tb_adults["15+"] = ( + tb_adults["total"] - tb_adults["0-4"].fillna(0) - tb_adults["5-9"].fillna(0) - tb_adults["10-14"].fillna(0) + ) + # Drop columns + tb_adults = tb_adults.drop(columns=["0-4", "5-9", "10-14", "total"]) + # Replace negative values for zero + flag = tb_adults["15+"] < 0 + tb_adults.loc[flag, "15+"] = 0 + # Shape table + tb_adults = tb_adults.melt(id_vars=cols_index, value_name="pop", var_name="age") + # Concatenate with original table + tb = pr.concat([tb, tb_adults], ignore_index=True) + return tb + + +def get_index_columns(tb): + cols_index = list(tb.columns.intersection(COLUMNS_INDEX)) + return cols_index + + +def add_prop(tb): + # Add + tbx = tb[tb["education"] == "total"].drop(columns=["education", "assr"]) + tb = tb.merge(tbx, on=["country", "year", "age", "sex", "scenario"], suffixes=["", "_total"]) + tb["prop"] = (100 * tb["pop"] / tb["pop_total"]).copy_metadata(tb["pop"]) + tb = tb.drop(columns=["pop_total"]) + return tb diff --git a/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital.meta.yml b/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital.meta.yml new file mode 100644 index 00000000000..f886cbc03b4 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital.meta.yml @@ -0,0 +1,530 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + display: + numDecimalPlaces: 2 + others: + education: |- + <> + title: |- + << title >> - Scenario: SSP<> + title_age: |- + << title >> - Scenario: SSP<> - + <%- if age == 'total' %> + All population + <%- else %> + Age: <> + <%- endif %> + title_sex: |- + << title >> - Scenario: SSP<> - + <%- if sex == 'total' %> + All population + <%- else %> + Sex: <>s + <%- endif %> + title_edu: |- + << title >> - Scenario: SSP<> - + <%- if education == 'total' %> + All population + <%- elif education == 'no_education' %> + Education: None + <%- else %> + Education: {definitions.others.education} + <%- endif %> + title_sex_age: |- + << title >> - Scenario: SSP<> - + <%- if (sex == 'total') and (age == 'total') %> + All population + <% else %> + <%- if (sex == 'total') %> + Sex: All - + <% else %> + Sex: <>s - + <%- endif %> + <%- if (age == 'total') %> + Age: All + <% else %> + Age: <> + <%- endif %> + <%- endif %> + title_age_edu: |- + << title >> - Scenario: SSP<> - + <%- if (age == 'total') and (education == 'total') %> + All population + <% else %> + <%- if (age == 'total') %> + Age: All - + <% else %> + Age: <> - + <%- endif %> + <%- if (education == 'total') %> + Education: All + <% elif (education == "no_education") %> + Education: None + <% else %> + Education: {definitions.others.education} + <%- endif %> + <%- endif %> + title_sex_age_edu: |- + << title >> - Scenario: SSP<> - + <%- if (age == 'total') and (sex == 'total') and (education == 'total') %> + All population + <% else %> + <%- if (sex == 'total') %> + Sex: All - + <% else %> + Sex: <>s - + <%- endif %> + <%- if (age == 'total') %> + Age: All - + <% else %> + Age: <> - + <%- endif %> + <%- if (education == 'total') %> + Education: All + <% elif (education == "no_education") %> + Education: None + <% else %> + Education: {definitions.others.education} + <%- endif %> + <%- endif %> + projections: |- + Projections from 2025 onwards are based on the << scenario >> scenario + dim_ssp: |- + <% if scenario == "1" %> + **Population Component of Rapid Development (SSP1)**: This scenario assumes a future that is moving toward a more sustainable path, with educational and health investments accelerating the demographic transition, leading to a relatively low world population. The emphasis is on strengthening human wellbeing. This is associated with high education, low mortality and low fertility. Migration levels are assumed to be medium for all countries under this SSP. + <%- elif scenario == "2" %> + **Population Component of Medium (SSP2)**: This is the middle of the road scenario that can also be seen as the most likely path for each country. It combines for all countries medium fertility with medium mortality, medium migration, and the Global Education Trend (GET) education scenario. + <%- elif scenario == "3" %> + **Population Component of Stalled Development (SSP3)**: This scenario portrays a world separated into regions characterized by extreme poverty, pockets of moderate wealth, and many countries struggling to maintain living standards for rapidly growing populations. This is associated with low education, high mortality and high fertility. Due to the emphasis on security and barriers to international exchange, migration is assumed to be low for all countries. + <%- elif scenario == "4" %> + **Population Component of Inequality (SSP4)**: This scenario assumes increasing inequalities and stratification both across and within countries. It assumes high fertility and mortality scenarios in high-fertility countries and low fertility and medium mortality scenarios in low-fertility countries combined with medium migration scenario. In terms of education the SSP3 education transitions (CER) are decreased by 10% for levels up to lower secondary education, keeping the SSP2 (GET) education transitions for upper and post-secondary education. + <%- elif scenario == "5" %> + **Population Component of Conventional Development (SSP5)**: This scenario is similar to SSP1 for the fertility, mortality and education assumptions except that it combines them with high migration scenario in a context of widespread globalization. + <%- endif %> + dim_edu: |- + <% if education == "total" %> + Over all education categories. + <% elif education == "under_15" %> + Population below 15 years old. + <% elif education == "no_education" %> + Highest level of education attained: No level and Grade 1 of ISCED 1 not completed. + <% elif education == "incomplete_primary" %> + Highest level of education attained: Incomplete ISCED 1. + <% elif education == "primary" %> + Highest level of education attained: Completed ISCED 1 and incomplete ISCED 2. + <% elif education == "lower_secondary" %> + Highest level of education attained: Completed ISCED 2 and incomplete ISCED 3. + <% elif education == "upper_secondary" %> + Highest level of education attained: Completed ISCED 3 and incomplete ISCED 4 5 or 6. + <% elif education == "post_secondary" %> + Highest level of education attained: Completed ISCED 4 5 6 7 or 8. + <% endif %> + dim_edu_all: |- + Under-15: Population below 15 years old. + No education: Highest level of education attained: No level and Grade 1 of ISCED 1 not completed. + Incomplete primary: Highest level of education attained: Incomplete ISCED 1. + Primary: Highest level of education attained: Completed ISCED 1 and incomplete ISCED 2. + Lower secondary: Highest level of education attained: Completed ISCED 2 and incomplete ISCED 3. + Upper secondary: Highest level of education attained: Completed ISCED 3 and incomplete ISCED 4 5 or 6. + Post secondary: Highest level of education attained: Completed ISCED 4 5 6 7 or 8. +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + update_period_days: 730 + title: Wittgenstein Centre, Human Capital (Historical data + Projections) + +tables: + main: + common: + description_key: + - "{definitions.others.dim_ssp}" + + variables: + # Population + growth: + title: |- + <% set title = "Average Annual Growth Rate" %> + {definitions.others.title} + unit: "%" + short_unit: "%" + description_short: |- + Rate of growth of the population over the period of one year. {definitions.others.projections}. + presentation: + topic_tags: + - Population Growth + nirate: + title: |- + <% set title = "Average Annual Rate of Natural Increase" %> + {definitions.others.title} + unit: "per 1,000 people" + description_short: |- + Represents the portion of population growth (or decline) determined exclusively by births and deaths. {definitions.others.projections}. + display: + numDecimalPlaces: 1 + presentation: + topic_tags: + - Population Growth + + # Years + ggapmys15: + title: |- + <% set title = "Gender Gap in Mean Years Schooling (15+)" %> + {definitions.others.title} + unit: years + description_short: |- + The difference between female and male mean years of schooling for the population over 15 years of age. {definitions.others.projections}. Negative values indicate that women spent fewer years in school. + display: + numDecimalPlaces: 0 + presentation: + topic_tags: + - Global Education + ggapmys25: + title: |- + <% set title = "Gender Gap in Mean Years Schooling (25+)" %> + {definitions.others.title} + unit: years + description_short: |- + The difference between female and male mean years of schooling for population over 25 years of age. {definitions.others.projections}. Negative values indicate that women spent fewer years in school. + display: + numDecimalPlaces: 0 + presentation: + topic_tags: + - Global Education + mage: + title: |- + <% set title = "Population Median Age" %> + {definitions.others.title} + unit: years + description_short: |- + Age that divides the population in two parts of equal size, that is, there are as many persons with ages above the median as there are with ages below the median. {definitions.others.projections}. + presentation: + topic_tags: + - Population Growth + + # Rates (Births, Deaths) + cbr: + title: |- + <% set title = "Crude Birth Rate" %> + {definitions.others.title} + unit: births per 1,000 people + description_short: |- + Number of live births recorded over the last 5-year period, divided by population-years. It is based on the civil calendar year, spanning from January 1 to December 31. {definitions.others.projections}. + presentation: + topic_tags: + - Fertility Rate + - Population Growth + + cdr: + title: |- + <% set title = "Crude Death Rate" %> + {definitions.others.title} + unit: deaths per 1,000 people + description_short: |- + Number of deaths recorded in the last 5-year period, divided by population-years. It is based on the civil calendar year, spanning from January 1 to December 31. {definitions.others.projections}. + presentation: + topic_tags: + - Population Growth + - Life Expectancy + + # Ratios + tdr: + title: |- + <% set title = "Total Dependency Ratio" %> + {definitions.others.title} + unit: "%" + description_short: |- + The ratio of the youth (0-14) and elderly (65+) population over the working age (15-64) population. {definitions.others.projections}. + description_processing: |- + The source gives the values as the number of young and elderly persons per person of working age. We have scaled the figures to have percentages instead. + presentation: + topic_tags: + - Age Structure + ydr: + title: |- + <% set title = "Youth Dependency Ratio" %> + {definitions.others.title} + unit: "%" + description_short: |- + The ratio of the youth (0-14) population over the working age (15-64) population. {definitions.others.projections}. + description_processing: |- + The source gives the values as the number of young persons per person of working age. We have scaled the figures to have percentages instead. + presentation: + topic_tags: + - Age Structure + + odr: + title: |- + <% set title = "Old-age Dependency Ratio" %> + {definitions.others.title} + unit: "%" + description_short: |- + The ratio of the elderly (65+) population over the working age (15-64) population. {definitions.others.projections}. + description_processing: |- + The source gives the values as the number of elderly persons per person of working age. We have scaled the figures to have percentages instead. + presentation: + topic_tags: + - Age Structure + + by_age: + common: + description_key: + - "{definitions.others.dim_ssp}" + + variables: + sexratio: + title: |- + <% set title = "Sex ratio" %> + {definitions.others.title_age} + unit: "males per 100 females" + description_short: |- + <% if age == "total" %> + The number of male population per 100 female population. {definitions.others.projections}. + <%- else %> + The number of male population per 100 female population (aged << age >>). {definitions.others.projections}. + <%- endif %> + presentation: + topic_tags: + - Gender Ratio + + by_sex: + common: + description_key: + - "{definitions.others.dim_ssp}" + + variables: + e0: + title: |- + <% set title = "Life expectancy" %> + {definitions.others.title_sex} + unit: "years" + description_short: |- + <% if sex == "total" %> + The number of years a newborn is expected to live. {definitions.others.projections}. + <%- else %> + The number of years a << sex >> newborn is expected to live. {definitions.others.projections}. + <% endif %> + presentation: + topic_tags: + - Life Expectancy + ryl15: + title: |- + <% set title = "Age when remaining Life Expectancy is below 15 years" %> + {definitions.others.title_sex} + unit: "years" + description_short: |- + <% if sex == "total" %> + The age of a person in a population when remaining life expectancy is 15 years. {definitions.others.projections}. + <%- else %> + The age of a << sex >> person in a population when remaining life expectancy is 15 years. {definitions.others.projections}. + <%- endif %> + description_processing: |- + Source: [Lutz et al. 2008](https://previous.iiasa.ac.at/web/home/research/researchPrograms/WorldPopulation/PublicationsMediaCoverage/POPNETNewsletter/popnet39.pdf#page=4) + presentation: + topic_tags: + - Life Expectancy + pryl15: + title: |- + <% set title = "Proportion of population with a remaining Life Expectancy below 15 Years" %> + {definitions.others.title_sex} + unit: "%" + short_unit: "%" + description_short: |- + <% if sex == "total" %> + Proportion of the population with a remaining life expectancy below 15 years in the total population. {definitions.others.projections}. + <%- else %> + Proportion of the << sex >> population with a remaining life expectancy below 15 years in the total population. {definitions.others.projections}. + <%- endif %> + description_processing: |- + Source: [Sanderson and Scherbov 2010](https://dataexplorer.wittgensteincentre.org/wcde-v3/www.iiasa.ac.at/web/home/research/researchPrograms/WorldPopulation/PublicationsMediaCoverage/POPNETNewsletter/popnet42.pdf#page=6) + presentation: + topic_tags: + - Life Expectancy + + by_edu: + variables: + ggapedu15: + title: |- + <% set title = "Gender gap in Educational Attainment (15+)" %> + {definitions.others.title_edu} + unit: "%" + short_unit: "%" + description_short: |- + The difference between the proportion of females and the proportion of males in the population over 15 years of age achieving education level "{definitions.others.education}". {definitions.others.projections}. + presentation: + topic_tags: + - Global Education + ggapedu25: + title: |- + <% set title = "Gender gap in Educational Attainment (25+)" %> + {definitions.others.title_edu} + unit: "%" + short_unit: "%" + description_short: |- + The difference between the proportion of females and the proportion of males in the population over 25 years of age achieving education level "{definitions.others.education}". {definitions.others.projections}. + presentation: + topic_tags: + - Global Education + # macb: + # title: |- + # <% set title = "Mean age at childbearing" %> + # {definitions.others.title_edu} + # unit: "years" + # short_unit: "years" + # description_short: |- + # <% if education == "total" %> + # The mean age of mothers at the birth of their children observed in the last five-year period. {definitions.others.projections}. + # <%- else %> + # The mean age of mothers at the birth of their children observed in the last five-year period. Data for people with education level "{definitions.others.education}". {definitions.others.projections}. + # <%- endif %> + # presentation: + # topic_tags: + # - Fertility Rate + tfr: + title: |- + <% set title = "Total Fertility Rate" %> + {definitions.others.title_edu} + unit: "births per woman in last 5 years" + description_short: |- + <% if education == "total" %> + The average number of children born in the last 5-year period to the women of reproductive age (15-49). + <%- else %> + The average number of children born in the last 5-year period to the women of reproductive age (15-49). Data for people with education level "{definitions.others.education}". {definitions.others.projections}. + <%- endif %> + presentation: + topic_tags: + - Fertility Rate + # net: + # title: |- + # <% set title = "Net migration" %> + # {definitions.others.title_edu} + # unit: "people" + # description_short: |- + # <% if education == "total" %> + # The number of immigrants minus the number of emigrants over the last five-year period. {definitions.others.projections}. + # <%- else %> + # The number of immigrants minus the number of emigrants over the last five-year period. Data for migrants with education level "{definitions.others.education}". {definitions.others.projections}. + # <%- endif %> + # presentation: + # topic_tags: + # - Migration + + by_sex_age: + common: + description_key: + - "{definitions.others.dim_ssp}" + + variables: + mys: + title: |- + <% set title = "Mean years of schooling" %> + {definitions.others.title_sex_age} + unit: years + description_short: |- + <%- if (age == "total") and (sex == "total") %> + Mean number of years spent in school. {definitions.others.projections}. + <%- elif (age != "total") and (sex == "total") %> + Mean number of years spent in school (aged <>). {definitions.others.projections}. + <%- elif (age == "total") and (sex != "total") %> + Mean number of years spent in school (<>s). {definitions.others.projections}. + <%- else %> + Mean number of years spent in school (aged <>, <>s). {definitions.others.projections}. + <%- endif %> + presentation: + topic_tags: + - Global Education + + by_age_edu: + common: + description_key: + - "{definitions.others.dim_ssp}" + + variables: + asfr: + title: |- + <% set title = "Age-Specific Fertility Rate" %> + {definitions.others.title_age_edu} + unit: births per 1,000 women + description_short: |- + <%- if (education == "total") %> + Number of births occurring to women aged <> in the last five years, divided by the number of women in the age group. {definitions.others.projections}. + <%- elif (education == "no_education") %> + Number of births occurring to women aged <> and no education in the last five years, divided by the number of women in that same group. {definitions.others.projections}. + <%- else %> + Number of births occurring to women aged <> and {definitions.others.education}-level education in the last five years, divided by the number of women in that age group. {definitions.others.projections}. + <%- endif %> + presentation: + topic_tags: + - Global Education + - Fertility Rate + + by_sex_age_edu: + common: + description_key: + - "{definitions.others.dim_ssp}" + + variables: + assr: + title: |- + <% set title = "Age-Specific Survival Ratio" %> + {definitions.others.title_sex_age_edu} + unit: "%" + description_short: |- + <%- if (education == "total") %> + <%- if (age == "newborn") %> + The percentage of the <> newborn who are alive after the last five-year period. {definitions.others.projections}. + <%- else %> + The percentage of the <> population aged <> who are alive after the last five-year period. {definitions.others.projections}. + <%- endif %> + <%- elif (education == "no_education") %> + The percentage of the <> population aged <> with no education who are alive after the last five-year period. {definitions.others.projections}. + <%- else %> + The percentage of the <> population aged <> with <> education who are alive after the last five-year period. {definitions.others.projections}. + <%- endif %> + presentation: + topic_tags: + - Life Expectancy + + pop: + title: |- + <% set title = "Population" %> + {definitions.others.title_sex_age_edu} + unit: "people" + description_short: |- + De facto population + <%- if (sex != "total") %> + of <>s + <%- endif %> + <%- if (age != "total") %> + aged <> + <%- endif %> + <%- if (education != "total") %> + with <%- if (education == "no_education") %>no education<%- else %><> education<%- endif %> + <%- endif %> + in a country or region. + presentation: + topic_tags: + - Population Growth + + prop: + title: |- + <% set title = "Educational Attainment Distribution" %> + {definitions.others.title_sex_age_edu} + unit: "%" + description_short: |- + De facto share of population + <%- if (sex != "total") %> + of <>s + <%- endif %> + <%- if (age != "total") %> + aged <> + <%- endif %> + <%- if (education != "total") %> + with <%- if (education == "no_education") %>no education<%- elif (education == "some_education") %>some formal education<%- else %><> education<%- endif %> + <%- endif %> + in a country or region. + description_processing: |- + We have estimated this indicator with the population values from this source. + presentation: + topic_tags: + - Global Education diff --git a/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital.py b/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital.py new file mode 100644 index 00000000000..1e218d25d32 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital.py @@ -0,0 +1,127 @@ +"""Load a meadow dataset and create a garden dataset.""" + +import owid.catalog.processing as pr + +from etl.helpers import PathFinder, create_dataset + +from .shared import get_index_columns + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) +# Not all columns are present in historical and projections datasets. This dictionary contains the expected differences. +TABLE_COLUMN_DIFFERENCES = { + "by_edu": { + "missing_in_hist": {"macb", "net"}, + }, + "by_sex_age": { + "missing_in_proj": {"net"}, + }, + "main": { + "missing_in_hist": {"emi", "imm"}, + "missing_in_proj": {"macb"}, + }, +} +DTYPES = { + "sex": "category", + "age": "category", + "education": "category", + "country": "category", + "year": "UInt16", + "scenario": "UInt8", +} + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_proj = paths.load_dataset("wittgenstein_human_capital_proj") + ds_hist = paths.load_dataset("wittgenstein_human_capital_historical") + + # Read table from meadow dataset. + paths.log.info("reading tables...") + tbs_proj = {t.m.short_name: t.reset_index() for t in ds_proj} + tbs_hist = {t.m.short_name: t.reset_index() for t in ds_hist} + + # + # Processing + # + assert tbs_proj.keys() == tbs_hist.keys(), "Mismatch in tables between historical and projection datasets" + + tables = [] + for key in tbs_proj.keys(): + paths.log.info(f"Building {key}") + + # Get tables + tb_proj = tbs_proj[key] + tb_hist = tbs_hist[key] + + # Dtypes + tb_proj = tb_proj.astype({k: v for k, v in DTYPES.items() if k in tb_proj.columns}) + tb_hist = tb_hist.astype({k: v for k, v in DTYPES.items() if k in tb_hist.columns}) + + # Check + sanity_checks(tb_proj, tb_hist) + + # Keep only the columns that are present in both datasets + columns_common = tb_proj.columns.intersection(tb_hist.columns) + + # Concatenate + tb = pr.concat([tb_proj[columns_common], tb_hist[columns_common]], ignore_index=True) + + # Remove duplicates + index = get_index_columns(tb) + tb = tb.drop_duplicates(subset=index, keep="first") + + # Format + tb = tb.format(index, short_name=key) + + # Reduce origins + for col in tb.columns: + tb[col].metadata.origins = [tb[col].metadata.origins[0]] + + # Add to list + tables.append(tb) + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, + tables=tables, + check_variables_metadata=True, + ) + + # Save changes in the new garden dataset. + ds_garden.save() + + +def sanity_checks(tb_proj, tb_hist): + # Short name sanity check + assert ( + tb_proj.m.short_name == tb_hist.m.short_name + ), f"Mismatch in short_name of historical ({tb_hist.m.short_name}) and projection ({tb_proj.m.short_name})" + key = tb_proj.m.short_name + + # Look for differences + missing_in_hist = set(tb_proj.columns) - set(tb_hist.columns) + missing_in_proj = set(tb_hist.columns) - set(tb_proj.columns) + + # Check with expected differences + if key in TABLE_COLUMN_DIFFERENCES: + missing_in_hist_expected = TABLE_COLUMN_DIFFERENCES[key].get("missing_in_hist", set()) + missing_in_proj_expected = TABLE_COLUMN_DIFFERENCES[key].get("missing_in_proj", set()) + assert missing_in_hist == missing_in_hist_expected, ( + f"Table {key}: Missing columns in historical dataset. " + f"Expected: {missing_in_hist_expected}, Found: {missing_in_hist}" + ) + assert missing_in_proj == missing_in_proj_expected, ( + f"Table {key}: Missing columns in projection dataset. " + f"Expected: {missing_in_proj_expected}, Found: {missing_in_proj}" + ) + else: + assert set(tb_proj.columns) == set(tb_hist.columns), ( + f"Table {key}: Mismatch in columns between historical and projection. " + f"Projection columns: {tb_proj.columns.tolist()}, Historical columns: {tb_hist.columns.tolist()}" + ) diff --git a/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital_historical.countries.json b/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital_historical.countries.json new file mode 100644 index 00000000000..3b8bc22b510 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital_historical.countries.json @@ -0,0 +1,231 @@ +{ + "Afghanistan": "Afghanistan", + "Albania": "Albania", + "Algeria": "Algeria", + "Angola": "Angola", + "Antigua and Barbuda": "Antigua and Barbuda", + "Argentina": "Argentina", + "Armenia": "Armenia", + "Aruba": "Aruba", + "Australia": "Australia", + "Austria": "Austria", + "Azerbaijan": "Azerbaijan", + "Bahamas": "Bahamas", + "Bahrain": "Bahrain", + "Bangladesh": "Bangladesh", + "Barbados": "Barbados", + "Belarus": "Belarus", + "Belgium": "Belgium", + "Belize": "Belize", + "Benin": "Benin", + "Bhutan": "Bhutan", + "Bolivia (Plurinational State of)": "Bolivia", + "Bosnia and Herzegovina": "Bosnia and Herzegovina", + "Botswana": "Botswana", + "Brazil": "Brazil", + "Brunei Darussalam": "Brunei", + "Bulgaria": "Bulgaria", + "Burkina Faso": "Burkina Faso", + "Burundi": "Burundi", + "Cambodia": "Cambodia", + "Cameroon": "Cameroon", + "Canada": "Canada", + "Cape Verde": "Cape Verde", + "Central African Republic": "Central African Republic", + "Chad": "Chad", + "Chile": "Chile", + "China": "China", + "Colombia": "Colombia", + "Comoros": "Comoros", + "Congo": "Congo", + "Costa Rica": "Costa Rica", + "Cote d'Ivoire": "Cote d'Ivoire", + "Croatia": "Croatia", + "Cuba": "Cuba", + "Cura\u00e7ao": "Curacao", + "Cyprus": "Cyprus", + "Czech Republic": "Czechia", + "Democratic People's Republic of Korea": "North Korea", + "Democratic Republic of the Congo": "Democratic Republic of Congo", + "Denmark": "Denmark", + "Djibouti": "Djibouti", + "Dominican Republic": "Dominican Republic", + "Ecuador": "Ecuador", + "Egypt": "Egypt", + "El Salvador": "El Salvador", + "Equatorial Guinea": "Equatorial Guinea", + "Eritrea": "Eritrea", + "Estonia": "Estonia", + "Ethiopia": "Ethiopia", + "Fiji": "Fiji", + "Finland": "Finland", + "France": "France", + "French Guiana": "French Guiana", + "French Polynesia": "French Polynesia", + "Gabon": "Gabon", + "Gambia": "Gambia", + "Georgia": "Georgia", + "Germany": "Germany", + "Ghana": "Ghana", + "Greece": "Greece", + "Grenada": "Grenada", + "Guadeloupe": "Guadeloupe", + "Guam": "Guam", + "Guatemala": "Guatemala", + "Guinea": "Guinea", + "Guinea-Bissau": "Guinea-Bissau", + "Guyana": "Guyana", + "Haiti": "Haiti", + "Honduras": "Honduras", + "Hungary": "Hungary", + "Iceland": "Iceland", + "India": "India", + "Indonesia": "Indonesia", + "Iran (Islamic Republic of)": "Iran", + "Iraq": "Iraq", + "Ireland": "Ireland", + "Israel": "Israel", + "Italy": "Italy", + "Jamaica": "Jamaica", + "Japan": "Japan", + "Jordan": "Jordan", + "Kazakhstan": "Kazakhstan", + "Kenya": "Kenya", + "Kiribati": "Kiribati", + "Kuwait": "Kuwait", + "Kyrgyzstan": "Kyrgyzstan", + "Lao People's Democratic Republic": "Laos", + "Latvia": "Latvia", + "Lebanon": "Lebanon", + "Lesotho": "Lesotho", + "Liberia": "Liberia", + "Lithuania": "Lithuania", + "Luxembourg": "Luxembourg", + "Madagascar": "Madagascar", + "Malawi": "Malawi", + "Malaysia": "Malaysia", + "Maldives": "Maldives", + "Mali": "Mali", + "Malta": "Malta", + "Martinique": "Martinique", + "Mauritania": "Mauritania", + "Mauritius": "Mauritius", + "Mayotte": "Mayotte", + "Mexico": "Mexico", + "Micronesia": "Micronesia (UNSD)", + "Micronesia (Federated States of)": "Micronesia (country)", + "Mongolia": "Mongolia", + "Montenegro": "Montenegro", + "Morocco": "Morocco", + "Mozambique": "Mozambique", + "Myanmar": "Myanmar", + "Namibia": "Namibia", + "Nepal": "Nepal", + "Netherlands": "Netherlands", + "New Caledonia": "New Caledonia", + "New Zealand": "New Zealand", + "Nicaragua": "Nicaragua", + "Niger": "Niger", + "Nigeria": "Nigeria", + "Norway": "Norway", + "Occupied Palestinian Territory": "Palestine", + "Oman": "Oman", + "Pakistan": "Pakistan", + "Panama": "Panama", + "Papua New Guinea": "Papua New Guinea", + "Paraguay": "Paraguay", + "Peru": "Peru", + "Philippines": "Philippines", + "Poland": "Poland", + "Portugal": "Portugal", + "Puerto Rico": "Puerto Rico", + "Qatar": "Qatar", + "Republic of Korea": "South Korea", + "Republic of Moldova": "Moldova", + "Reunion": "Reunion", + "Romania": "Romania", + "Russian Federation": "Russia", + "Rwanda": "Rwanda", + "Saint Lucia": "Saint Lucia", + "Saint Vincent and the Grenadines": "Saint Vincent and the Grenadines", + "Samoa": "Samoa", + "Sao Tome and Principe": "Sao Tome and Principe", + "Saudi Arabia": "Saudi Arabia", + "Senegal": "Senegal", + "Serbia": "Serbia", + "Seychelles": "Seychelles", + "Sierra Leone": "Sierra Leone", + "Singapore": "Singapore", + "Slovakia": "Slovakia", + "Slovenia": "Slovenia", + "Solomon Islands": "Solomon Islands", + "Somalia": "Somalia", + "South Africa": "South Africa", + "South Sudan": "South Sudan", + "Spain": "Spain", + "Sri Lanka": "Sri Lanka", + "Sudan": "Sudan", + "Suriname": "Suriname", + "Swaziland": "Eswatini", + "Sweden": "Sweden", + "Switzerland": "Switzerland", + "Syrian Arab Republic": "Syria", + "Tajikistan": "Tajikistan", + "Thailand": "Thailand", + "Timor-Leste": "East Timor", + "Togo": "Togo", + "Tonga": "Tonga", + "Trinidad and Tobago": "Trinidad and Tobago", + "Tunisia": "Tunisia", + "Turkey": "Turkey", + "Turkmenistan": "Turkmenistan", + "Uganda": "Uganda", + "Ukraine": "Ukraine", + "United Arab Emirates": "United Arab Emirates", + "United Kingdom of Great Britain and Northern Ireland": "United Kingdom", + "United Republic of Tanzania": "Tanzania", + "United States Virgin Islands": "United States Virgin Islands", + "United States of America": "United States", + "Uruguay": "Uruguay", + "Uzbekistan": "Uzbekistan", + "Vanuatu": "Vanuatu", + "Venezuela (Bolivarian Republic of)": "Venezuela", + "Viet Nam": "Vietnam", + "Western Sahara": "Western Sahara", + "World": "World", + "Yemen": "Yemen", + "Zambia": "Zambia", + "Zimbabwe": "Zimbabwe", + "Africa": "Africa (UNSD)", + "Asia": "Asia (UNSD)", + "Australia and New Zealand": "Australia and New Zealand (UNSD)", + "Caribbean": "Caribbean (UNSD)", + "Central America": "Central America (UNSD)", + "Central Asia": "Central Asia (UNSD)", + "Eastern Africa": "Eastern Africa (UNSD)", + "Eastern Asia": "Eastern Asia (UNSD)", + "Eastern Europe": "Eastern Europe (UNSD)", + "Europe": "Europe (UNSD)", + "Hong Kong Special Administrative Region of China": "Hong Kong", + "Latin America and the Caribbean": "Latin America and the Caribbean (UNSD)", + "Libyan Arab Jamahiriya": "Libya", + "Macao Special Administrative Region of China": "Macao", + "Melanesia": "Melanesia (UNSD)", + "Middle Africa": "Middle Africa (UNSD)", + "Northern Africa": "Northern Africa (UNSD)", + "Northern America": "Northern America (UNSD)", + "Northern Europe": "Northern Europe (UNSD)", + "Oceania": "Oceania (UNSD)", + "Polynesia": "Polynesia (UNSD)", + "South America": "South America (UNSD)", + "South-Eastern Asia": "South-eastern Asia (UNSD)", + "Southern Africa": "Southern Africa (UNSD)", + "Southern Asia": "Southern Asia (UNSD)", + "Southern Europe": "Southern Europe (UNSD)", + "Taiwan Province of China": "Taiwan", + "The former Yugoslav Republic of Macedonia": "North Macedonia", + "Western Africa": "Western Africa (UNSD)", + "Western Asia": "Western Asia (UNSD)", + "Western Europe": "Western Europe (UNSD)", + "Channel Islands": "Channel Islands" +} diff --git a/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital_historical.meta.yml b/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital_historical.meta.yml new file mode 100644 index 00000000000..fbddd08a368 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital_historical.meta.yml @@ -0,0 +1,501 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + display: + numDecimalPlaces: 2 + others: + education: |- + <> + title: |- + << title >> - Scenario: SSP<> + title_age: |- + << title >> - Scenario: SSP<> - + <%- if age == 'total' %> + All population + <%- else %> + Age: <> + <%- endif %> + title_sex: |- + << title >> - Scenario: SSP<> - + <%- if sex == 'total' %> + All population + <%- else %> + Sex: <>s + <%- endif %> + title_edu: |- + << title >> - Scenario: SSP<> - + <%- if education == 'total' %> + All population + <%- elif education == 'no_education' %> + Education: None + <%- else %> + Education: {definitions.others.education} + <%- endif %> + title_sex_age: |- + << title >> - Scenario: SSP<> - + <%- if (sex == 'total') and (age == 'total') %> + All population + <% else %> + <%- if (sex == 'total') %> + Sex: All - + <% else %> + Sex: <>s - + <%- endif %> + <%- if (age == 'total') %> + Age: All + <% else %> + Age: <> + <%- endif %> + <%- endif %> + title_age_edu: |- + << title >> - Scenario: SSP<> - + <%- if (age == 'total') and (education == 'total') %> + All population + <% else %> + <%- if (age == 'total') %> + Age: All - + <% else %> + Age: <> - + <%- endif %> + <%- if (education == 'total') %> + Education: All + <% elif (education == "no_education") %> + Education: None + <% else %> + Education: {definitions.others.education} + <%- endif %> + <%- endif %> + title_sex_age_edu: |- + << title >> - Scenario: SSP<> - + <%- if (age == 'total') and (sex == 'total') and (education == 'total') %> + All population + <% else %> + <%- if (sex == 'total') %> + Sex: All - + <% else %> + Sex: <>s - + <%- endif %> + <%- if (age == 'total') %> + Age: All - + <% else %> + Age: <> - + <%- endif %> + <%- if (education == 'total') %> + Education: All + <% elif (education == "no_education") %> + Education: None + <% else %> + Education: {definitions.others.education} + <%- endif %> + <%- endif %> + projections: |- + Projections from 2025 onwards are based on the << scenario >> scenario + dim_ssp: |- + <% if scenario == "1" %> + **Population Component of Rapid Development (SSP1)**: This scenario assumes a future that is moving toward a more sustainable path, with educational and health investments accelerating the demographic transition, leading to a relatively low world population. The emphasis is on strengthening human wellbeing. This is associated with high education, low mortality and low fertility. Migration levels are assumed to be medium for all countries under this SSP. + <%- elif scenario == "2" %> + **Population Component of Medium (SSP2)**: This is the middle of the road scenario that can also be seen as the most likely path for each country. It combines for all countries medium fertility with medium mortality, medium migration, and the Global Education Trend (GET) education scenario. + <%- elif scenario == "3" %> + **Population Component of Stalled Development (SSP3)**: This scenario portrays a world separated into regions characterized by extreme poverty, pockets of moderate wealth, and many countries struggling to maintain living standards for rapidly growing populations. This is associated with low education, high mortality and high fertility. Due to the emphasis on security and barriers to international exchange, migration is assumed to be low for all countries. + <%- elif scenario == "4" %> + **Population Component of Inequality (SSP4)**: This scenario assumes increasing inequalities and stratification both across and within countries. It assumes high fertility and mortality scenarios in high-fertility countries and low fertility and medium mortality scenarios in low-fertility countries combined with medium migration scenario. In terms of education the SSP3 education transitions (CER) are decreased by 10% for levels up to lower secondary education, keeping the SSP2 (GET) education transitions for upper and post-secondary education. + <%- elif scenario == "5" %> + **Population Component of Conventional Development (SSP5)**: This scenario is similar to SSP1 for the fertility, mortality and education assumptions except that it combines them with high migration scenario in a context of widespread globalization. + <%- endif %> + dim_edu: |- + <% if education == "total" %> + Over all education categories. + <% elif education == "under_15" %> + Population below 15 years old. + <% elif education == "no_education" %> + Highest level of education attained: No level and Grade 1 of ISCED 1 not completed. + <% elif education == "incomplete_primary" %> + Highest level of education attained: Incomplete ISCED 1. + <% elif education == "primary" %> + Highest level of education attained: Completed ISCED 1 and incomplete ISCED 2. + <% elif education == "lower_secondary" %> + Highest level of education attained: Completed ISCED 2 and incomplete ISCED 3. + <% elif education == "upper_secondary" %> + Highest level of education attained: Completed ISCED 3 and incomplete ISCED 4 5 or 6. + <% elif education == "post_secondary" %> + Highest level of education attained: Completed ISCED 4 5 6 7 or 8. + <% endif %> + dim_edu_all: |- + Under-15: Population below 15 years old. + No education: Highest level of education attained: No level and Grade 1 of ISCED 1 not completed. + Incomplete primary: Highest level of education attained: Incomplete ISCED 1. + Primary: Highest level of education attained: Completed ISCED 1 and incomplete ISCED 2. + Lower secondary: Highest level of education attained: Completed ISCED 2 and incomplete ISCED 3. + Upper secondary: Highest level of education attained: Completed ISCED 3 and incomplete ISCED 4 5 or 6. + Post secondary: Highest level of education attained: Completed ISCED 4 5 6 7 or 8. +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + update_period_days: 730 + title: Wittgenstein Centre, Human Capital (Historical data) + +tables: + main: + common: + description_key: + - "{definitions.others.dim_ssp}" + + variables: + # Population + growth: + title: |- + <% set title = "Average Annual Growth Rate" %> + {definitions.others.title} + unit: "%" + short_unit: "%" + description_short: |- + Rate of growth of the population over the period of one year. {definitions.others.projections}. + presentation: + topic_tags: + - Population Growth + nirate: + title: |- + <% set title = "Average Annual Rate of Natural Increase" %> + {definitions.others.title} + unit: "per 1,000 people" + description_short: |- + Represents the portion of population growth (or decline) determined exclusively by births and deaths. {definitions.others.projections}. + display: + numDecimalPlaces: 1 + presentation: + topic_tags: + - Population Growth + + # Years + ggapmys15: + title: |- + <% set title = "Gender Gap in Mean Years Schooling (15+)" %> + {definitions.others.title} + unit: years + description_short: |- + The difference between female and male mean years of schooling for the population over 15 years of age. {definitions.others.projections}. Negative values indicate that women spent fewer years in school. + display: + numDecimalPlaces: 0 + presentation: + topic_tags: + - Global Education + ggapmys25: + title: |- + <% set title = "Gender Gap in Mean Years Schooling (25+)" %> + {definitions.others.title} + unit: years + description_short: |- + The difference between female and male mean years of schooling for population over 25 years of age. {definitions.others.projections}. Negative values indicate that women spent fewer years in school. + display: + numDecimalPlaces: 0 + presentation: + topic_tags: + - Global Education + mage: + title: |- + <% set title = "Population Median Age" %> + {definitions.others.title} + unit: years + description_short: |- + Age that divides the population in two parts of equal size, that is, there are as many persons with ages above the median as there are with ages below the median. {definitions.others.projections}. + presentation: + topic_tags: + - Population Growth + + # Rates (Births, Deaths) + cbr: + title: |- + <% set title = "Crude Birth Rate" %> + {definitions.others.title} + unit: births per 1,000 people + description_short: |- + Number of live births recorded over the last 5-year period, divided by population-years. It is based on the civil calendar year, spanning from January 1 to December 31. {definitions.others.projections}. + presentation: + topic_tags: + - Fertility Rate + - Population Growth + + cdr: + title: |- + <% set title = "Crude Death Rate" %> + {definitions.others.title} + unit: deaths per 1,000 people + description_short: |- + Number of deaths recorded in the last 5-year period, divided by population-years. It is based on the civil calendar year, spanning from January 1 to December 31. {definitions.others.projections}. + presentation: + topic_tags: + - Population Growth + - Life Expectancy + + # Ratios + tdr: + title: |- + <% set title = "Total Dependency Ratio" %> + {definitions.others.title} + unit: "%" + description_short: |- + The ratio of the youth (0-14) and elderly (65+) population over the working age (15-64) population. {definitions.others.projections}. + description_processing: |- + The source gives the values as the number of young and elderly persons per person of working age. We have scaled the figures to have percentages instead. + presentation: + topic_tags: + - Age Structure + ydr: + title: |- + <% set title = "Youth Dependency Ratio" %> + {definitions.others.title} + unit: "%" + description_short: |- + The ratio of the youth (0-14) population over the working age (15-64) population. {definitions.others.projections}. + description_processing: |- + The source gives the values as the number of young persons per person of working age. We have scaled the figures to have percentages instead. + presentation: + topic_tags: + - Age Structure + + odr: + title: |- + <% set title = "Old-age Dependency Ratio" %> + {definitions.others.title} + unit: "%" + description_short: |- + The ratio of the elderly (65+) population over the working age (15-64) population. {definitions.others.projections}. + description_processing: |- + The source gives the values as the number of elderly persons per person of working age. We have scaled the figures to have percentages instead. + presentation: + topic_tags: + - Age Structure + + by_age: + variables: + sexratio: + title: |- + <% set title = "Sex ratio" %> + {definitions.others.title_age} + unit: "males per 100 females" + description_short: |- + <% if age == "total" %> + The number of male population per 100 female population. {definitions.others.projections}. + <%- else %> + The number of male population per 100 female population (aged << age >>). {definitions.others.projections}. + <%- endif %> + presentation: + topic_tags: + - Gender Ratio + + by_sex: + variables: + e0: + title: |- + <% set title = "Life expectancy" %> + {definitions.others.title_sex} + unit: "years" + description_short: |- + <% if sex == "total" %> + The number of years a newborn is expected to live. {definitions.others.projections}. + <%- else %> + The number of years a << sex >> newborn is expected to live. {definitions.others.projections}. + <% endif %> + presentation: + topic_tags: + - Life Expectancy + ryl15: + title: |- + <% set title = "Age when remaining Life Expectancy is below 15 years" %> + {definitions.others.title_sex} + unit: "years" + description_short: |- + <% if sex == "total" %> + The age of a person in a population when remaining life expectancy is 15 years. {definitions.others.projections}. + <%- else %> + The age of a << sex >> person in a population when remaining life expectancy is 15 years. {definitions.others.projections}. + <%- endif %> + description_processing: |- + Source: [Lutz et al. 2008](https://previous.iiasa.ac.at/web/home/research/researchPrograms/WorldPopulation/PublicationsMediaCoverage/POPNETNewsletter/popnet39.pdf#page=4) + presentation: + topic_tags: + - Life Expectancy + pryl15: + title: |- + <% set title = "Proportion of population with a remaining Life Expectancy below 15 Years" %> + {definitions.others.title_sex} + unit: "%" + short_unit: "%" + description_short: |- + <% if sex == "total" %> + Proportion of the population with a remaining life expectancy below 15 years in the total population. {definitions.others.projections}. + <%- else %> + Proportion of the << sex >> population with a remaining life expectancy below 15 years in the total population. {definitions.others.projections}. + <%- endif %> + description_processing: |- + Source: [Sanderson and Scherbov 2010](https://dataexplorer.wittgensteincentre.org/wcde-v3/www.iiasa.ac.at/web/home/research/researchPrograms/WorldPopulation/PublicationsMediaCoverage/POPNETNewsletter/popnet42.pdf#page=6) + presentation: + topic_tags: + - Life Expectancy + + by_edu: + variables: + ggapedu15: + title: |- + <% set title = "Gender gap in Educational Attainment (15+)" %> + {definitions.others.title_edu} + unit: "%" + short_unit: "%" + description_short: |- + The difference between the proportion of females and the proportion of males in the population over 15 years of age achieving education level "{definitions.others.education}". {definitions.others.projections}. + presentation: + topic_tags: + - Global Education + ggapedu25: + title: |- + <% set title = "Gender gap in Educational Attainment (25+)" %> + {definitions.others.title_edu} + unit: "%" + short_unit: "%" + description_short: |- + The difference between the proportion of females and the proportion of males in the population over 25 years of age achieving education level "{definitions.others.education}". {definitions.others.projections}. + presentation: + topic_tags: + - Global Education + + tfr: + title: |- + <% set title = "Total Fertility Rate" %> + {definitions.others.title_edu} + unit: "births per woman in last 5 years" + description_short: |- + <% if education == "total" %> + The average number of children born in the last 5-year period to the women of reproductive age (15-49). + <%- else %> + The average number of children born in the last 5-year period to the women of reproductive age (15-49). Data for people with education level "{definitions.others.education}". {definitions.others.projections}. + <%- endif %> + presentation: + topic_tags: + - Fertility Rate + + by_sex_age: + variables: + mys: + title: |- + <% set title = "Mean years of schooling" %> + {definitions.others.title_sex_age} + unit: years + description_short: |- + <%- if (age == "total") and (sex == "total") %> + Mean number of years spent in school. {definitions.others.projections}. + <%- elif (age != "total") and (sex == "total") %> + Mean number of years spent in school (aged <>). {definitions.others.projections}. + <%- elif (age == "total") and (sex != "total") %> + Mean number of years spent in school (<>s). {definitions.others.projections}. + <%- else %> + Mean number of years spent in school (aged <>, <>s). {definitions.others.projections}. + <%- endif %> + presentation: + topic_tags: + - Global Education + + net: + title: |- + <% set title = "Net migration" %> + {definitions.others.title_sex_age} + unit: "people" + description_short: |- + <%- if (age == "total") and (sex == "total") %> + The number of immigrants minus the number of emigrants over the last five-year period. {definitions.others.projections}. + <%- elif (age != "total") and (sex == "total") %> + The number of immigrants minus the number of emigrants over the last five-year period. Only migrants aged <> are considered. {definitions.others.projections}. + <%- elif (age == "total") and (sex != "total") %> + The number of immigrants minus the number of emigrants over the last five-year period. Only <> migrants are considered. {definitions.others.projections}. + <%- else %> + The number of immigrants minus the number of emigrants over the last five-year period. Only <> migrants aged <> are considered. {definitions.others.projections}. + <%- endif %> + presentation: + topic_tags: + - Migration + + by_age_edu: + variables: + asfr: + title: |- + <% set title = "Age-Specific Fertility Rate" %> + {definitions.others.title_age_edu} + unit: births per 1,000 women + description_short: |- + <%- if (education == "total") %> + Number of births occurring to women aged <> in the last five years, divided by the number of women in the age group. {definitions.others.projections}. + <%- elif (education == "no_education") %> + Number of births occurring to women aged <> and no education in the last five years, divided by the number of women in that same group. {definitions.others.projections}. + <%- else %> + Number of births occurring to women aged <> and {definitions.others.education}-level education in the last five years, divided by the number of women in that age group. {definitions.others.projections}. + <%- endif %> + presentation: + topic_tags: + - Global Education + - Fertility Rate + + by_sex_age_edu: + variables: + assr: + title: |- + <% set title = "Age-Specific Survival Ratio" %> + {definitions.others.title_sex_age_edu} + unit: "%" + description_short: |- + <%- if (education == "total") %> + <%- if (age == "newborn") %> + The percentage of the <> newborn who are alive after the last five-year period. {definitions.others.projections}. + <%- else %> + The percentage of the <> population aged <> who are alive after the last five-year period. {definitions.others.projections}. + <%- endif %> + <%- elif (education == "no_education") %> + The percentage of the <> population aged <> with no education who are alive after the last five-year period. {definitions.others.projections}. + <%- else %> + The percentage of the <> population aged <> with <> education who are alive after the last five-year period. {definitions.others.projections}. + <%- endif %> + presentation: + topic_tags: + - Life Expectancy + + pop: + title: |- + <% set title = "Population" %> + {definitions.others.title_sex_age_edu} + unit: "people" + description_short: |- + De facto population + <%- if (sex != "total") %> + of <>s + <%- endif %> + <%- if (age != "total") %> + aged <> + <%- endif %> + <%- if (education != "total") %> + with <%- if (education == "no_education") %>no education<%- else %><> education<%- endif %> + <%- endif %> + in a country or region. + presentation: + topic_tags: + - Population Growth + + prop: + title: |- + <% set title = "Educational Attainment Distribution" %> + {definitions.others.title_sex_age_edu} + unit: "%" + description_short: |- + De facto share of population + <%- if (sex != "total") %> + of <>s + <%- endif %> + <%- if (age != "total") %> + aged <> + <%- endif %> + <%- if (education != "total") %> + with <%- if (education == "no_education") %>no education<%- if (education == "some_education") %>some formal education<%- else %><> education<%- endif %> + <%- endif %> + in a country or region. + description_processing: |- + We have estimated this indicator with the population values from this source. + presentation: + topic_tags: + - Global Education diff --git a/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital_historical.py b/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital_historical.py new file mode 100644 index 00000000000..3eaa19284ce --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital_historical.py @@ -0,0 +1,136 @@ +"""Load a meadow dataset and create a garden dataset.""" + +from etl.helpers import PathFinder, create_dataset + +from .shared import add_dim_15plus, add_dim_some_education, add_prop, make_table + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("wittgenstein_human_capital_historical") + + # Read table from meadow dataset. + paths.log.info("reading tables...") + tb = ds_meadow.read("main").reset_index(drop=True) + tb_age = ds_meadow.read("by_age").reset_index(drop=True) + tb_sex = ds_meadow.read("by_sex").reset_index(drop=True) + tb_edu = ds_meadow.read("by_edu").reset_index( + drop=True + ) # TODO: add metadata field for table explaining the different education levels + tb_sex_age = ds_meadow.read("by_sex_age").reset_index(drop=True) + tb_age_edu = ds_meadow.read("by_age_edu").reset_index(drop=True) + tb_sex_age_edu = ds_meadow.read("by_sex_age_edu").reset_index(drop=True) + + # + # Process data. + # + + # 1/ MAKE MAIN TABLE + paths.log.info("baking main table...") + tb = make_table( + tb, + country_mapping_path=paths.country_mapping_path, + cols_single=["ggapmys25", "ggapmys15", "mage", "odr", "ydr", "tdr"], + cols_range=["nirate", "cbr", "cdr", "macb", "growth"], + per_100=["tdr", "odr", "ydr"], + ) + + # 2.1/ MAKE BY AGE TABLE (sexratio) + paths.log.info("baking tables by age...") + tb_age = make_table( + tb_age, + country_mapping_path=paths.country_mapping_path, + all_single=True, + per_100=["sexratio"], + ) + + # 2.2/ BY SEX + paths.log.info("baking tables by sex...") + tb_sex = make_table( + tb_sex, + country_mapping_path=paths.country_mapping_path, + all_range=True, + ) + + # 2.3/ BY EDU + paths.log.info("baking tables by education...") + tb_edu = make_table( + tb_edu, + country_mapping_path=paths.country_mapping_path, + cols_single=["ggapedu15", "ggapedu25"], + cols_range=["tfr"], + ) + + # 3.1/ BY SEX+AGE + paths.log.info("baking tables by sex+age...") + tb_sex_age = make_table( + tb_sex_age, + country_mapping_path=paths.country_mapping_path, + cols_single=["mys"], + cols_range=["net"], + per_100=["net"], + ) + + # 3.2/ BY AGE+EDU + paths.log.info("baking tables by age+education...") + assert "total" not in set(tb_age_edu["age"].unique()), "Unexpected age category: 'total'" + tb_age_edu = make_table( + tb_age_edu, + country_mapping_path=paths.country_mapping_path, + all_range=True, + ) + + # 4.1/ BY SEX+AGE+EDU + paths.log.info("baking tables by sex+age+education...") + tb_sex_age_edu = make_table( + tb_sex_age_edu, + country_mapping_path=paths.country_mapping_path, + dtypes={ + "sex": "category", + "age": "category", + "education": "category", + }, + cols_single=["pop"], + cols_range=["assr"], + per_1000=["pop"], + per_100=["assr"], + ) + + # Add education="some_education" (only for sex=total and age=total, and indicator 'pop') + tb_sex_age_edu = add_dim_some_education(tb_sex_age_edu) + + # Add 15+ age group + tb_sex_age_edu = add_dim_15plus(tb_sex_age_edu) + + # Add population share + tb_sex_age_edu = add_prop(tb_sex_age_edu) + + # + # Save outputs. + # + # Format + tables = [ + tb.format(["country", "year", "scenario"], short_name="main"), + tb_age.format(["country", "scenario", "age", "year"], short_name="by_age"), + tb_sex.format(["country", "scenario", "sex", "year"], short_name="by_sex"), + tb_edu.format(["country", "scenario", "education", "year"], short_name="by_edu"), + tb_sex_age.format(["country", "scenario", "sex", "age", "year"], short_name="by_sex_age"), + tb_age_edu.format(["country", "scenario", "age", "education", "year"], short_name="by_age_edu"), + tb_sex_age_edu.format(["country", "scenario", "sex", "age", "education", "year"], short_name="by_sex_age_edu"), + ] + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, + tables=tables, + check_variables_metadata=True, + default_metadata=ds_meadow.metadata, + ) + + # Save changes in the new garden dataset. + ds_garden.save() diff --git a/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital_proj.countries.json b/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital_proj.countries.json new file mode 100644 index 00000000000..3b8bc22b510 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital_proj.countries.json @@ -0,0 +1,231 @@ +{ + "Afghanistan": "Afghanistan", + "Albania": "Albania", + "Algeria": "Algeria", + "Angola": "Angola", + "Antigua and Barbuda": "Antigua and Barbuda", + "Argentina": "Argentina", + "Armenia": "Armenia", + "Aruba": "Aruba", + "Australia": "Australia", + "Austria": "Austria", + "Azerbaijan": "Azerbaijan", + "Bahamas": "Bahamas", + "Bahrain": "Bahrain", + "Bangladesh": "Bangladesh", + "Barbados": "Barbados", + "Belarus": "Belarus", + "Belgium": "Belgium", + "Belize": "Belize", + "Benin": "Benin", + "Bhutan": "Bhutan", + "Bolivia (Plurinational State of)": "Bolivia", + "Bosnia and Herzegovina": "Bosnia and Herzegovina", + "Botswana": "Botswana", + "Brazil": "Brazil", + "Brunei Darussalam": "Brunei", + "Bulgaria": "Bulgaria", + "Burkina Faso": "Burkina Faso", + "Burundi": "Burundi", + "Cambodia": "Cambodia", + "Cameroon": "Cameroon", + "Canada": "Canada", + "Cape Verde": "Cape Verde", + "Central African Republic": "Central African Republic", + "Chad": "Chad", + "Chile": "Chile", + "China": "China", + "Colombia": "Colombia", + "Comoros": "Comoros", + "Congo": "Congo", + "Costa Rica": "Costa Rica", + "Cote d'Ivoire": "Cote d'Ivoire", + "Croatia": "Croatia", + "Cuba": "Cuba", + "Cura\u00e7ao": "Curacao", + "Cyprus": "Cyprus", + "Czech Republic": "Czechia", + "Democratic People's Republic of Korea": "North Korea", + "Democratic Republic of the Congo": "Democratic Republic of Congo", + "Denmark": "Denmark", + "Djibouti": "Djibouti", + "Dominican Republic": "Dominican Republic", + "Ecuador": "Ecuador", + "Egypt": "Egypt", + "El Salvador": "El Salvador", + "Equatorial Guinea": "Equatorial Guinea", + "Eritrea": "Eritrea", + "Estonia": "Estonia", + "Ethiopia": "Ethiopia", + "Fiji": "Fiji", + "Finland": "Finland", + "France": "France", + "French Guiana": "French Guiana", + "French Polynesia": "French Polynesia", + "Gabon": "Gabon", + "Gambia": "Gambia", + "Georgia": "Georgia", + "Germany": "Germany", + "Ghana": "Ghana", + "Greece": "Greece", + "Grenada": "Grenada", + "Guadeloupe": "Guadeloupe", + "Guam": "Guam", + "Guatemala": "Guatemala", + "Guinea": "Guinea", + "Guinea-Bissau": "Guinea-Bissau", + "Guyana": "Guyana", + "Haiti": "Haiti", + "Honduras": "Honduras", + "Hungary": "Hungary", + "Iceland": "Iceland", + "India": "India", + "Indonesia": "Indonesia", + "Iran (Islamic Republic of)": "Iran", + "Iraq": "Iraq", + "Ireland": "Ireland", + "Israel": "Israel", + "Italy": "Italy", + "Jamaica": "Jamaica", + "Japan": "Japan", + "Jordan": "Jordan", + "Kazakhstan": "Kazakhstan", + "Kenya": "Kenya", + "Kiribati": "Kiribati", + "Kuwait": "Kuwait", + "Kyrgyzstan": "Kyrgyzstan", + "Lao People's Democratic Republic": "Laos", + "Latvia": "Latvia", + "Lebanon": "Lebanon", + "Lesotho": "Lesotho", + "Liberia": "Liberia", + "Lithuania": "Lithuania", + "Luxembourg": "Luxembourg", + "Madagascar": "Madagascar", + "Malawi": "Malawi", + "Malaysia": "Malaysia", + "Maldives": "Maldives", + "Mali": "Mali", + "Malta": "Malta", + "Martinique": "Martinique", + "Mauritania": "Mauritania", + "Mauritius": "Mauritius", + "Mayotte": "Mayotte", + "Mexico": "Mexico", + "Micronesia": "Micronesia (UNSD)", + "Micronesia (Federated States of)": "Micronesia (country)", + "Mongolia": "Mongolia", + "Montenegro": "Montenegro", + "Morocco": "Morocco", + "Mozambique": "Mozambique", + "Myanmar": "Myanmar", + "Namibia": "Namibia", + "Nepal": "Nepal", + "Netherlands": "Netherlands", + "New Caledonia": "New Caledonia", + "New Zealand": "New Zealand", + "Nicaragua": "Nicaragua", + "Niger": "Niger", + "Nigeria": "Nigeria", + "Norway": "Norway", + "Occupied Palestinian Territory": "Palestine", + "Oman": "Oman", + "Pakistan": "Pakistan", + "Panama": "Panama", + "Papua New Guinea": "Papua New Guinea", + "Paraguay": "Paraguay", + "Peru": "Peru", + "Philippines": "Philippines", + "Poland": "Poland", + "Portugal": "Portugal", + "Puerto Rico": "Puerto Rico", + "Qatar": "Qatar", + "Republic of Korea": "South Korea", + "Republic of Moldova": "Moldova", + "Reunion": "Reunion", + "Romania": "Romania", + "Russian Federation": "Russia", + "Rwanda": "Rwanda", + "Saint Lucia": "Saint Lucia", + "Saint Vincent and the Grenadines": "Saint Vincent and the Grenadines", + "Samoa": "Samoa", + "Sao Tome and Principe": "Sao Tome and Principe", + "Saudi Arabia": "Saudi Arabia", + "Senegal": "Senegal", + "Serbia": "Serbia", + "Seychelles": "Seychelles", + "Sierra Leone": "Sierra Leone", + "Singapore": "Singapore", + "Slovakia": "Slovakia", + "Slovenia": "Slovenia", + "Solomon Islands": "Solomon Islands", + "Somalia": "Somalia", + "South Africa": "South Africa", + "South Sudan": "South Sudan", + "Spain": "Spain", + "Sri Lanka": "Sri Lanka", + "Sudan": "Sudan", + "Suriname": "Suriname", + "Swaziland": "Eswatini", + "Sweden": "Sweden", + "Switzerland": "Switzerland", + "Syrian Arab Republic": "Syria", + "Tajikistan": "Tajikistan", + "Thailand": "Thailand", + "Timor-Leste": "East Timor", + "Togo": "Togo", + "Tonga": "Tonga", + "Trinidad and Tobago": "Trinidad and Tobago", + "Tunisia": "Tunisia", + "Turkey": "Turkey", + "Turkmenistan": "Turkmenistan", + "Uganda": "Uganda", + "Ukraine": "Ukraine", + "United Arab Emirates": "United Arab Emirates", + "United Kingdom of Great Britain and Northern Ireland": "United Kingdom", + "United Republic of Tanzania": "Tanzania", + "United States Virgin Islands": "United States Virgin Islands", + "United States of America": "United States", + "Uruguay": "Uruguay", + "Uzbekistan": "Uzbekistan", + "Vanuatu": "Vanuatu", + "Venezuela (Bolivarian Republic of)": "Venezuela", + "Viet Nam": "Vietnam", + "Western Sahara": "Western Sahara", + "World": "World", + "Yemen": "Yemen", + "Zambia": "Zambia", + "Zimbabwe": "Zimbabwe", + "Africa": "Africa (UNSD)", + "Asia": "Asia (UNSD)", + "Australia and New Zealand": "Australia and New Zealand (UNSD)", + "Caribbean": "Caribbean (UNSD)", + "Central America": "Central America (UNSD)", + "Central Asia": "Central Asia (UNSD)", + "Eastern Africa": "Eastern Africa (UNSD)", + "Eastern Asia": "Eastern Asia (UNSD)", + "Eastern Europe": "Eastern Europe (UNSD)", + "Europe": "Europe (UNSD)", + "Hong Kong Special Administrative Region of China": "Hong Kong", + "Latin America and the Caribbean": "Latin America and the Caribbean (UNSD)", + "Libyan Arab Jamahiriya": "Libya", + "Macao Special Administrative Region of China": "Macao", + "Melanesia": "Melanesia (UNSD)", + "Middle Africa": "Middle Africa (UNSD)", + "Northern Africa": "Northern Africa (UNSD)", + "Northern America": "Northern America (UNSD)", + "Northern Europe": "Northern Europe (UNSD)", + "Oceania": "Oceania (UNSD)", + "Polynesia": "Polynesia (UNSD)", + "South America": "South America (UNSD)", + "South-Eastern Asia": "South-eastern Asia (UNSD)", + "Southern Africa": "Southern Africa (UNSD)", + "Southern Asia": "Southern Asia (UNSD)", + "Southern Europe": "Southern Europe (UNSD)", + "Taiwan Province of China": "Taiwan", + "The former Yugoslav Republic of Macedonia": "North Macedonia", + "Western Africa": "Western Africa (UNSD)", + "Western Asia": "Western Asia (UNSD)", + "Western Europe": "Western Europe (UNSD)", + "Channel Islands": "Channel Islands" +} diff --git a/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital_proj.meta.yml b/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital_proj.meta.yml new file mode 100644 index 00000000000..02a41a28d00 --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital_proj.meta.yml @@ -0,0 +1,565 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + display: + numDecimalPlaces: 2 + others: + education: |- + <> + title: |- + << title >> - Scenario: SSP<> + title_age: |- + << title >> - Scenario: SSP<> - + <%- if age == 'total' %> + All population + <%- else %> + Age: <> + <%- endif %> + title_sex: |- + << title >> - Scenario: SSP<> - + <%- if sex == 'total' %> + All population + <%- else %> + Sex: <>s + <%- endif %> + title_edu: |- + << title >> - Scenario: SSP<> - + <%- if education == 'total' %> + All population + <%- elif education == 'no_education' %> + Education: None + <%- else %> + Education: {definitions.others.education} + <%- endif %> + title_sex_age: |- + << title >> - Scenario: SSP<> - + <%- if (sex == 'total') and (age == 'total') %> + All population + <% else %> + <%- if (sex == 'total') %> + Sex: All - + <% else %> + Sex: <>s - + <%- endif %> + <%- if (age == 'total') %> + Age: All + <% else %> + Age: <> + <%- endif %> + <%- endif %> + title_age_edu: |- + << title >> - Scenario: SSP<> - + <%- if (age == 'total') and (education == 'total') %> + All population + <% else %> + <%- if (age == 'total') %> + Age: All - + <% else %> + Age: <> - + <%- endif %> + <%- if (education == 'total') %> + Education: All + <% elif (education == "no_education") %> + Education: None + <% else %> + Education: {definitions.others.education} + <%- endif %> + <%- endif %> + title_sex_age_edu: |- + << title >> - Scenario: SSP<> - + <%- if (age == 'total') and (sex == 'total') and (education == 'total') %> + All population + <% else %> + <%- if (sex == 'total') %> + Sex: All - + <% else %> + Sex: <>s - + <%- endif %> + <%- if (age == 'total') %> + Age: All - + <% else %> + Age: <> - + <%- endif %> + <%- if (education == 'total') %> + Education: All + <% elif (education == "no_education") %> + Education: None + <% else %> + Education: {definitions.others.education} + <%- endif %> + <%- endif %> + projections: |- + Projections from 2025 onwards are based on the << scenario >> scenario + dim_ssp: |- + <% if scenario == "1" %> + **Population Component of Rapid Development (SSP1)**: This scenario assumes a future that is moving toward a more sustainable path, with educational and health investments accelerating the demographic transition, leading to a relatively low world population. The emphasis is on strengthening human wellbeing. This is associated with high education, low mortality and low fertility. Migration levels are assumed to be medium for all countries under this SSP. + <%- elif scenario == "2" %> + **Population Component of Medium (SSP2)**: This is the middle of the road scenario that can also be seen as the most likely path for each country. It combines for all countries medium fertility with medium mortality, medium migration, and the Global Education Trend (GET) education scenario. + <%- elif scenario == "3" %> + **Population Component of Stalled Development (SSP3)**: This scenario portrays a world separated into regions characterized by extreme poverty, pockets of moderate wealth, and many countries struggling to maintain living standards for rapidly growing populations. This is associated with low education, high mortality and high fertility. Due to the emphasis on security and barriers to international exchange, migration is assumed to be low for all countries. + <%- elif scenario == "4" %> + **Population Component of Inequality (SSP4)**: This scenario assumes increasing inequalities and stratification both across and within countries. It assumes high fertility and mortality scenarios in high-fertility countries and low fertility and medium mortality scenarios in low-fertility countries combined with medium migration scenario. In terms of education the SSP3 education transitions (CER) are decreased by 10% for levels up to lower secondary education, keeping the SSP2 (GET) education transitions for upper and post-secondary education. + <%- elif scenario == "5" %> + **Population Component of Conventional Development (SSP5)**: This scenario is similar to SSP1 for the fertility, mortality and education assumptions except that it combines them with high migration scenario in a context of widespread globalization. + <%- endif %> + dim_edu: |- + <% if education == "total" %> + Over all education categories. + <% elif education == "under_15" %> + Population below 15 years old. + <% elif education == "no_education" %> + Highest level of education attained: No level and Grade 1 of ISCED 1 not completed. + <% elif education == "incomplete_primary" %> + Highest level of education attained: Incomplete ISCED 1. + <% elif education == "primary" %> + Highest level of education attained: Completed ISCED 1 and incomplete ISCED 2. + <% elif education == "lower_secondary" %> + Highest level of education attained: Completed ISCED 2 and incomplete ISCED 3. + <% elif education == "upper_secondary" %> + Highest level of education attained: Completed ISCED 3 and incomplete ISCED 4 5 or 6. + <% elif education == "post_secondary" %> + Highest level of education attained: Completed ISCED 4 5 6 7 or 8. + <% endif %> + dim_edu_all: |- + Under-15: Population below 15 years old. + No education: Highest level of education attained: No level and Grade 1 of ISCED 1 not completed. + Incomplete primary: Highest level of education attained: Incomplete ISCED 1. + Primary: Highest level of education attained: Completed ISCED 1 and incomplete ISCED 2. + Lower secondary: Highest level of education attained: Completed ISCED 2 and incomplete ISCED 3. + Upper secondary: Highest level of education attained: Completed ISCED 3 and incomplete ISCED 4 5 or 6. + Post secondary: Highest level of education attained: Completed ISCED 4 5 6 7 or 8. +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + update_period_days: 730 + title: Wittgenstein Centre, Human Capital (Projections) + +tables: + main: + common: + description_key: + - "{definitions.others.dim_ssp}" + + variables: + # Population + growth: + title: |- + <% set title = "Average Annual Growth Rate" %> + {definitions.others.title} + unit: "%" + short_unit: "%" + description_short: |- + Rate of growth of the population over the period of one year. {definitions.others.projections}. + presentation: + topic_tags: + - Population Growth + nirate: + title: |- + <% set title = "Average Annual Rate of Natural Increase" %> + {definitions.others.title} + unit: "per 1,000 people" + description_short: |- + Represents the portion of population growth (or decline) determined exclusively by births and deaths. {definitions.others.projections}. + display: + numDecimalPlaces: 1 + presentation: + topic_tags: + - Population Growth + + # Years + ggapmys15: + title: |- + <% set title = "Gender Gap in Mean Years Schooling (15+)" %> + {definitions.others.title} + unit: years + description_short: |- + The difference between female and male mean years of schooling for the population over 15 years of age. {definitions.others.projections}. Negative values indicate that women spent fewer years in school. + display: + numDecimalPlaces: 0 + presentation: + topic_tags: + - Global Education + ggapmys25: + title: |- + <% set title = "Gender Gap in Mean Years Schooling (25+)" %> + {definitions.others.title} + unit: years + description_short: |- + The difference between female and male mean years of schooling for population over 25 years of age. {definitions.others.projections}. Negative values indicate that women spent fewer years in school. + display: + numDecimalPlaces: 0 + presentation: + topic_tags: + - Global Education + mage: + title: |- + <% set title = "Population Median Age" %> + {definitions.others.title} + unit: years + description_short: |- + Age that divides the population in two parts of equal size, that is, there are as many persons with ages above the median as there are with ages below the median. {definitions.others.projections}. + presentation: + topic_tags: + - Population Growth + + # Rates (Births, Deaths) + cbr: + title: |- + <% set title = "Crude Birth Rate" %> + {definitions.others.title} + unit: births per 1,000 people + description_short: |- + Number of live births recorded over the last 5-year period, divided by population-years. It is based on the civil calendar year, spanning from January 1 to December 31. {definitions.others.projections}. + presentation: + topic_tags: + - Fertility Rate + - Population Growth + + cdr: + title: |- + <% set title = "Crude Death Rate" %> + {definitions.others.title} + unit: deaths per 1,000 people + description_short: |- + Number of deaths recorded in the last 5-year period, divided by population-years. It is based on the civil calendar year, spanning from January 1 to December 31. {definitions.others.projections}. + presentation: + topic_tags: + - Population Growth + - Life Expectancy + + # Ratios + tdr: + title: |- + <% set title = "Total Dependency Ratio" %> + {definitions.others.title} + unit: "%" + description_short: |- + The ratio of the youth (0-14) and elderly (65+) population over the working age (15-64) population. {definitions.others.projections}. + description_processing: |- + The source gives the values as the number of young and elderly persons per person of working age. We have scaled the figures to have percentages instead. + presentation: + topic_tags: + - Age Structure + ydr: + title: |- + <% set title = "Youth Dependency Ratio" %> + {definitions.others.title} + unit: "%" + description_short: |- + The ratio of the youth (0-14) population over the working age (15-64) population. {definitions.others.projections}. + description_processing: |- + The source gives the values as the number of young persons per person of working age. We have scaled the figures to have percentages instead. + presentation: + topic_tags: + - Age Structure + + odr: + title: |- + <% set title = "Old-age Dependency Ratio" %> + {definitions.others.title} + unit: "%" + description_short: |- + The ratio of the elderly (65+) population over the working age (15-64) population. {definitions.others.projections}. + description_processing: |- + The source gives the values as the number of elderly persons per person of working age. We have scaled the figures to have percentages instead. + presentation: + topic_tags: + - Age Structure + + # Migration + emi: + title: |- + <% set title = "Emigration Flows" %> + {definitions.others.title} + unit: "people in last 5 years" + description_short: |- + The number of emigrants over a five-year period. {definitions.others.projections}. + description_processing: |- + The source provides the values in thousands. We have scaled them to have the actual number of emigrants. + display: + numDecimalPlaces: 0 + presentation: + topic_tags: + - Migration + + imm: + title: |- + <% set title = "Immigration Flows" %> + {definitions.others.title} + unit: "people in last 5 years" + description_short: |- + The number of immigrants over a five-year period. {definitions.others.projections}. + description_processing: |- + The source provides the values in thousands. We have scaled them to have the actual number of immigrants. + display: + numDecimalPlaces: 0 + presentation: + topic_tags: + - Migration + + by_age: + common: + description_key: + - "{definitions.others.dim_ssp}" + + variables: + sexratio: + title: |- + <% set title = "Sex ratio" %> + {definitions.others.title_age} + unit: "males per 100 females" + description_short: |- + <% if age == "total" %> + The number of male population per 100 female population. {definitions.others.projections}. + <%- else %> + The number of male population per 100 female population (aged << age >>). {definitions.others.projections}. + <%- endif %> + presentation: + topic_tags: + - Gender Ratio + + by_sex: + common: + description_key: + - "{definitions.others.dim_ssp}" + + variables: + e0: + title: |- + <% set title = "Life expectancy" %> + {definitions.others.title_sex} + unit: "years" + description_short: |- + <% if sex == "total" %> + The number of years a newborn is expected to live. {definitions.others.projections}. + <%- else %> + The number of years a << sex >> newborn is expected to live. {definitions.others.projections}. + <% endif %> + presentation: + topic_tags: + - Life Expectancy + ryl15: + title: |- + <% set title = "Age when remaining Life Expectancy is below 15 years" %> + {definitions.others.title_sex} + unit: "years" + description_short: |- + <% if sex == "total" %> + The age of a person in a population when remaining life expectancy is 15 years. {definitions.others.projections}. + <%- else %> + The age of a << sex >> person in a population when remaining life expectancy is 15 years. {definitions.others.projections}. + <%- endif %> + description_processing: |- + Source: [Lutz et al. 2008](https://previous.iiasa.ac.at/web/home/research/researchPrograms/WorldPopulation/PublicationsMediaCoverage/POPNETNewsletter/popnet39.pdf#page=4) + presentation: + topic_tags: + - Life Expectancy + pryl15: + title: |- + <% set title = "Proportion of population with a remaining Life Expectancy below 15 Years" %> + {definitions.others.title_sex} + unit: "%" + short_unit: "%" + description_short: |- + <% if sex == "total" %> + Proportion of the population with a remaining life expectancy below 15 years in the total population. {definitions.others.projections}. + <%- else %> + Proportion of the << sex >> population with a remaining life expectancy below 15 years in the total population. {definitions.others.projections}. + <%- endif %> + description_processing: |- + Source: [Sanderson and Scherbov 2010](https://dataexplorer.wittgensteincentre.org/wcde-v3/www.iiasa.ac.at/web/home/research/researchPrograms/WorldPopulation/PublicationsMediaCoverage/POPNETNewsletter/popnet42.pdf#page=6) + presentation: + topic_tags: + - Life Expectancy + + by_edu: + common: + description_key: + - "{definitions.others.dim_ssp}" + + variables: + ggapedu15: + title: |- + <% set title = "Gender gap in Educational Attainment (15+)" %> + {definitions.others.title_edu} + unit: "%" + short_unit: "%" + description_short: |- + The difference between the proportion of females and the proportion of males in the population over 15 years of age achieving education level "{definitions.others.education}". {definitions.others.projections}. + presentation: + topic_tags: + - Global Education + ggapedu25: + title: |- + <% set title = "Gender gap in Educational Attainment (25+)" %> + {definitions.others.title_edu} + unit: "%" + short_unit: "%" + description_short: |- + The difference between the proportion of females and the proportion of males in the population over 25 years of age achieving education level "{definitions.others.education}". {definitions.others.projections}. + presentation: + topic_tags: + - Global Education + macb: + title: |- + <% set title = "Mean age at childbearing" %> + {definitions.others.title_edu} + unit: "years" + short_unit: "years" + description_short: |- + <% if education == "total" %> + The mean age of mothers at the birth of their children observed in the last five-year period. {definitions.others.projections}. + <%- else %> + The mean age of mothers at the birth of their children observed in the last five-year period. Data for people with education level "{definitions.others.education}". {definitions.others.projections}. + <%- endif %> + presentation: + topic_tags: + - Fertility Rate + tfr: + title: |- + <% set title = "Total Fertility Rate" %> + {definitions.others.title_edu} + unit: "births per woman in last 5 years" + description_short: |- + <% if education == "total" %> + The average number of children born in the last 5-year period to the women of reproductive age (15-49). + <%- else %> + The average number of children born in the last 5-year period to the women of reproductive age (15-49). Data for people with education level "{definitions.others.education}". {definitions.others.projections}. + <%- endif %> + presentation: + topic_tags: + - Fertility Rate + net: + title: |- + <% set title = "Net migration" %> + {definitions.others.title_edu} + unit: "people" + description_short: |- + <% if education == "total" %> + The number of immigrants minus the number of emigrants over the last five-year period. {definitions.others.projections}. + <%- else %> + The number of immigrants minus the number of emigrants over the last five-year period. Data for migrants with education level "{definitions.others.education}". {definitions.others.projections}. + <%- endif %> + presentation: + topic_tags: + - Migration + + by_sex_age: + common: + description_key: + - "{definitions.others.dim_ssp}" + + variables: + mys: + title: |- + <% set title = "Mean years of schooling" %> + {definitions.others.title_sex_age} + unit: years + description_short: |- + <%- if (age == "total") and (sex == "total") %> + Mean number of years spent in school. {definitions.others.projections}. + <%- elif (age != "total") and (sex == "total") %> + Mean number of years spent in school (aged <>). {definitions.others.projections}. + <%- elif (age == "total") and (sex != "total") %> + Mean number of years spent in school (<>s). {definitions.others.projections}. + <%- else %> + Mean number of years spent in school (aged <>, <>s). {definitions.others.projections}. + <%- endif %> + presentation: + topic_tags: + - Global Education + + by_age_edu: + common: + description_key: + - "{definitions.others.dim_ssp}" + + variables: + asfr: + title: |- + <% set title = "Age-Specific Fertility Rate" %> + {definitions.others.title_age_edu} + unit: births per 1,000 women + description_short: |- + <%- if (education == "total") %> + Number of births occurring to women aged <> in the last five years, divided by the number of women in the age group. {definitions.others.projections}. + <%- elif (education == "no_education") %> + Number of births occurring to women aged <> and no education in the last five years, divided by the number of women in that same group. {definitions.others.projections}. + <%- else %> + Number of births occurring to women aged <> and {definitions.others.education}-level education in the last five years, divided by the number of women in that age group. {definitions.others.projections}. + <%- endif %> + presentation: + topic_tags: + - Global Education + - Fertility Rate + + by_sex_age_edu: + common: + description_key: + - "{definitions.others.dim_ssp}" + + variables: + assr: + title: |- + <% set title = "Age-Specific Survival Ratio" %> + {definitions.others.title_sex_age_edu} + unit: "%" + description_short: |- + <%- if (education == "total") %> + <%- if (age == "newborn") %> + The percentage of the <> newborn who are alive after the last five-year period. {definitions.others.projections}. + <%- else %> + The percentage of the <> population aged <> who are alive after the last five-year period. {definitions.others.projections}. + <%- endif %> + <%- elif (education == "no_education") %> + The percentage of the <> population aged <> with no education who are alive after the last five-year period. {definitions.others.projections}. + <%- else %> + The percentage of the <> population aged <> with <> education who are alive after the last five-year period. {definitions.others.projections}. + <%- endif %> + presentation: + topic_tags: + - Life Expectancy + + pop: + title: |- + <% set title = "Population" %> + {definitions.others.title_sex_age_edu} + unit: "people" + description_short: |- + De facto population + <%- if (sex != "total") %> + of <>s + <%- endif %> + <%- if (age != "total") %> + aged <> + <%- endif %> + <%- if (education != "total") %> + with <%- if (education == "no_education") %>no education<%- else %><> education<%- endif %> + <%- endif %> + in a country or region. + presentation: + topic_tags: + - Population Growth + + prop: + title: |- + <% set title = "Educational Attainment Distribution" %> + {definitions.others.title_sex_age_edu} + unit: "%" + description_short: |- + De facto share of population + <%- if (sex != "total") %> + of <>s + <%- endif %> + <%- if (age != "total") %> + aged <> + <%- endif %> + <%- if (education != "total") %> + with <%- if (education == "no_education") %>no education<%- elif (education == "some_education") %>some formal education<%- else %><> education<%- endif %> + <%- endif %> + in a country or region. + description_processing: |- + We have estimated this indicator with the population values from this source. + presentation: + topic_tags: + - Global Education diff --git a/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital_proj.py b/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital_proj.py new file mode 100644 index 00000000000..f5f51a679ce --- /dev/null +++ b/etl/steps/data/garden/demography/2024-12-06/wittgenstein_human_capital_proj.py @@ -0,0 +1,136 @@ +"""Load a meadow dataset and create a garden dataset.""" + +from etl.helpers import PathFinder, create_dataset + +from .shared import add_dim_15plus, add_dim_some_education, add_prop, make_table + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("wittgenstein_human_capital_proj") + + # Read table from meadow dataset. + paths.log.info("reading tables...") + tb = ds_meadow.read("main").reset_index(drop=True) + tb_age = ds_meadow.read("by_age").reset_index(drop=True) + tb_sex = ds_meadow.read("by_sex").reset_index(drop=True) + tb_edu = ds_meadow.read("by_edu").reset_index( + drop=True + ) # TODO: add metadata field for table explaining the different education levels + tb_sex_age = ds_meadow.read("by_sex_age").reset_index(drop=True) + tb_age_edu = ds_meadow.read("by_age_edu").reset_index(drop=True) + tb_sex_age_edu = ds_meadow.read("by_sex_age_edu").reset_index(drop=True) + + # + # Process data. + # + + # 1/ MAKE MAIN TABLE + paths.log.info("baking main table...") + tb = make_table( + tb, + country_mapping_path=paths.country_mapping_path, + cols_single=["tdr", "ggapmys25", "mage", "ydr", "ggapmys15", "odr"], + cols_range=["growth", "imm", "emi", "cbr", "nirate", "cdr"], + per_100=["tdr", "odr", "ydr"], + per_1000=["emi", "imm"], + ) + + # 2.1/ MAKE BY AGE TABLE (sexratio) + paths.log.info("baking tables by age...") + tb_age = make_table( + tb_age, + country_mapping_path=paths.country_mapping_path, + all_single=True, + per_100=["sexratio"], + ) + + # 2.2/ BY SEX + paths.log.info("baking tables by sex...") + tb_sex = make_table( + tb_sex, + country_mapping_path=paths.country_mapping_path, + all_range=True, + ) + + # 2.3/ BY EDU + paths.log.info("baking tables by education...") + tb_edu = make_table( + tb_edu, + country_mapping_path=paths.country_mapping_path, + cols_single=["ggapedu15", "ggapedu25"], + cols_range=["macb", "tfr", "net"], + per_1000=["net"], + ) + + # 3.1/ BY SEX+AGE + paths.log.info("baking tables by sex+age...") + tb_sex_age = make_table( + tb_sex_age, + country_mapping_path=paths.country_mapping_path, + all_single=True, + ) + + # 3.2/ BY AGE+EDU + paths.log.info("baking tables by age+education...") + assert "total" not in set(tb_age_edu["age"].unique()), "Unexpected age category: 'total'" + tb_age_edu = make_table( + tb_age_edu, + country_mapping_path=paths.country_mapping_path, + all_range=True, + ) + + # 4.1/ BY SEX+AGE+EDU + paths.log.info("baking tables by sex+age+education...") + tb_sex_age_edu = make_table( + tb_sex_age_edu, + country_mapping_path=paths.country_mapping_path, + dtypes={ + "sex": "category", + "age": "category", + "education": "category", + }, + cols_single=["pop"], + cols_range=["assr"], + per_1000=["pop"], + per_100=["assr"], + ) + + # Add education="some_education" (only for sex=total and age=total, and indicator 'pop') + tb_sex_age_edu = add_dim_some_education(tb_sex_age_edu) + + # Add 15+ age group + tb_sex_age_edu = add_dim_15plus(tb_sex_age_edu) + + # Add population share + tb_sex_age_edu = add_prop(tb_sex_age_edu) + + # + # Save outputs. + # + # Format + tables = [ + tb.format(["country", "year", "scenario"], short_name="main"), + tb_age.format(["country", "scenario", "age", "year"], short_name="by_age"), + tb_sex.format(["country", "scenario", "sex", "year"], short_name="by_sex"), + tb_edu.format(["country", "scenario", "education", "year"], short_name="by_edu"), + tb_sex_age.format(["country", "scenario", "sex", "age", "year"], short_name="by_sex_age"), + tb_age_edu.format(["country", "scenario", "age", "education", "year"], short_name="by_age_edu"), + tb_sex_age_edu.format(["country", "scenario", "sex", "age", "education", "year"], short_name="by_sex_age_edu"), + ] + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, + tables=tables, + check_variables_metadata=True, + default_metadata=ds_meadow.metadata, + ) + + # Save changes in the new garden dataset. + ds_garden.save() diff --git a/etl/steps/data/garden/education/2024-12-11/people_with_education.meta.yml b/etl/steps/data/garden/education/2024-12-11/people_with_education.meta.yml new file mode 100644 index 00000000000..b24e45ac0c9 --- /dev/null +++ b/etl/steps/data/garden/education/2024-12-11/people_with_education.meta.yml @@ -0,0 +1,43 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + processing_level: major + presentation: + topic_tags: + - Global Education + display: + numDecimalPlaces: 1 + short_unit: "%" + unit: "%" + description_key: + - Historical data for educational attainment between 1870 to 1950 comes from van Zanden, J. et al. (2014). + - "Data for 1950 to 2015 is sourced from the Wittgenstein Centre Human Capital Centre. These projections are based on collected census and survey data. The SSP2 is a middle-of-the-road scenario that combines medium fertility with medium mortality, medium migration, and the Global Education Trend (GET) education scenario. For more information and other projection models, consult the Wittgenstein Centre for Demography and Global Human Capital's website: https://dataexplorer.wittgensteincentre.org/." + - Data for 2020 onwards is also based on the Medium Shared Socioeconomic Pathways (SSP2) Wittgenstein Centre for Demography and Global Human Capital projections. For more information, see https://pure.iiasa.ac.at/id/eprint/19487/. + +dataset: + update_period_days: 365 + title: People with formal basic education (Wittgenstein Centre, OECD) + +tables: + people_with_education: + variables: + no_basic_education: + title: Share of population with no education + description_short: Share of people aged 15 or older who have not received some kind of formal [primary](#dod:primary-education), [secondary](#dod:secondary-education), or [tertiary](#dod:tertiary-education) education. + description_processing: |- + For each country and year, the share of the population aged 15 and older with no formal education was calculated. This involved summing up the population with no formal education and dividing it by the total population aged 15 and older for each country and year, then converting this ratio into a percentage. + + A global estimate was calculated for each year by summing the total population aged 15 and older across all countries and the total population within this age group with no formal education. The share of the global population aged 15+ with no formal education was then computed for each year. + + Historical data from van Zanden, J. et al. (2014) with estimates from 1870 to 1950 was combined with educational attainment estimates from Wittgenstein Centre for Demography and Global Human Capita. + basic_education: + title: Share of population with at least some basic education + description_short: Share of people aged 15 or older who have received at least some kind of formal [primary](#dod:primary-education), [secondary](#dod:secondary-education), or [tertiary](#dod:tertiary-education) education. + description_processing: |- + For each country and year, the share of the population aged 15 and older with no formal education was calculated. This involved summing up the population with no formal education and dividing it by the total population aged 15 and older for each country and year, then converting this ratio into a percentage. + + A global estimate was calculated for each year by summing the total population aged 15 and older across all countries and the total population within this age group with no formal education. The share of the global population aged 15+ with no formal education was then computed for each year. + + Historical data from van Zanden, J. et al. (2014) with estimates from 1870 to 1950 was combined with educational attainment estimates from Wittgenstein Centre for Demography and Global Human Capita. + + To calculate the share of the population with at least some basic education, the share of the population with no formal education was subtracted from 100%. diff --git a/etl/steps/data/garden/education/2024-12-11/people_with_education.py b/etl/steps/data/garden/education/2024-12-11/people_with_education.py new file mode 100644 index 00000000000..79974a37453 --- /dev/null +++ b/etl/steps/data/garden/education/2024-12-11/people_with_education.py @@ -0,0 +1,85 @@ +"""Load a meadow dataset and create a garden dataset.""" + +import owid.catalog.processing as pr + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load Wittgenstein data + ds_wc = paths.load_dataset("wittgenstein_human_capital") + tb_wc = ds_wc.read("by_sex_age_edu") + + # Load garden historical OECD dataset. + ds_oecd = paths.load_dataset("oecd_education") + tb_oecd = ds_oecd.read("oecd_education") + + # + # Process data. + # + # Prepare OECD + tb_oecd = make_oecd(tb_oecd) + countries_oecd = set(tb_oecd["country"].unique()) + + # Prepare Wittgenstein Center + tb_wc = make_wc(tb_wc) + countries_wc = set(tb_wc["country"].unique()) + + # Combine tables + tb = pr.concat([tb_oecd, tb_wc], short_name="education") + # Keep only relevant countries + countries = countries_oecd.intersection(countries_wc) + tb = tb.loc[tb["country"].isin(countries)] + # Format + tb = tb.format(["country", "year"], short_name="people_with_education") + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True) + + # Save changes in the new garden dataset. + ds_garden.save() + + +def make_oecd(tb): + # Filter the for years above 2020 (New Wittgenstein Center data starts at 2020) + tb = tb.loc[ + tb["year"] < 1950, ["country", "year", "no_formal_education", "population_with_basic_education"] + ].reset_index(drop=True) + + # Rename columns + tb = tb.rename( + columns={ + "no_formal_education": "no_basic_education", + "population_with_basic_education": "basic_education", + } + ) + return tb + + +def make_wc(tb): + tb = tb.loc[ + (tb["scenario"] == 2) + # & (tb_wc["country"] == "World") + & (tb["sex"] == "total") + & (tb["age"] == "15+") + & (tb["education"].isin(["no_education"])), + ["country", "year", "prop"], + ] + assert tb.groupby(["country", "year"]).size().max() == 1, "Only 1 rows per country-year accepted" + + # Estimate "no formal education" + tb = tb.rename(columns={"prop": "no_basic_education"}) + + # Estimate "with basic education" + tb["basic_education"] = 100 - tb["no_basic_education"] + + return tb diff --git a/etl/steps/data/garden/eia/2023-07-10/energy_consumption.py b/etl/steps/data/garden/eia/2023-07-10/energy_consumption.py index b469a4fb600..df380a0c938 100644 --- a/etl/steps/data/garden/eia/2023-07-10/energy_consumption.py +++ b/etl/steps/data/garden/eia/2023-07-10/energy_consumption.py @@ -1,6 +1,5 @@ -"""Garden step for EIA total energy consumption. +"""Garden step for EIA total energy consumption.""" -""" from typing import Dict, List, Optional, Union import numpy as np diff --git a/etl/steps/data/garden/eia/2023-12-12/energy_consumption.py b/etl/steps/data/garden/eia/2023-12-12/energy_consumption.py index f9cacca4947..b9eb9b3fd41 100644 --- a/etl/steps/data/garden/eia/2023-12-12/energy_consumption.py +++ b/etl/steps/data/garden/eia/2023-12-12/energy_consumption.py @@ -1,6 +1,4 @@ -"""Garden step for EIA total energy consumption. - -""" +"""Garden step for EIA total energy consumption.""" from etl.data_helpers import geo from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/garden/ember/2022-08-01/combined_electricity_review.py b/etl/steps/data/garden/ember/2022-08-01/combined_electricity_review.py index 794a8e6065f..f4659c720ca 100644 --- a/etl/steps/data/garden/ember/2022-08-01/combined_electricity_review.py +++ b/etl/steps/data/garden/ember/2022-08-01/combined_electricity_review.py @@ -1,6 +1,4 @@ -"""Garden step that combines Ember's European and Global Electricity Review. - -""" +"""Garden step that combines Ember's European and Global Electricity Review.""" import pandas as pd from owid import catalog diff --git a/etl/steps/data/garden/ember/2022-08-01/european_electricity_review.py b/etl/steps/data/garden/ember/2022-08-01/european_electricity_review.py index 3243535ad17..2dde2fee239 100644 --- a/etl/steps/data/garden/ember/2022-08-01/european_electricity_review.py +++ b/etl/steps/data/garden/ember/2022-08-01/european_electricity_review.py @@ -1,6 +1,4 @@ -"""Garden step for European Electricity Review (Ember, 2022). - -""" +"""Garden step for European Electricity Review (Ember, 2022).""" from typing import cast diff --git a/etl/steps/data/garden/ember/2022-08-01/global_electricity_review.py b/etl/steps/data/garden/ember/2022-08-01/global_electricity_review.py index 04a149cec9e..944dba801e8 100644 --- a/etl/steps/data/garden/ember/2022-08-01/global_electricity_review.py +++ b/etl/steps/data/garden/ember/2022-08-01/global_electricity_review.py @@ -1,6 +1,4 @@ -"""Garden step for Global Electricity Review (Ember, 2022). - -""" +"""Garden step for Global Electricity Review (Ember, 2022).""" import pandas as pd from owid import catalog diff --git a/etl/steps/data/garden/ember/2023-02-20/shared.py b/etl/steps/data/garden/ember/2023-02-20/shared.py index 92f3e7e08da..76b11ae037d 100644 --- a/etl/steps/data/garden/ember/2023-02-20/shared.py +++ b/etl/steps/data/garden/ember/2023-02-20/shared.py @@ -470,7 +470,7 @@ def add_region_aggregates( def _expand_combinations_in_amendments( - amendments: List[Tuple[Dict[Any, Any], Dict[Any, Any]]] + amendments: List[Tuple[Dict[Any, Any], Dict[Any, Any]]], ) -> List[Tuple[Dict[Any, Any], Dict[Any, Any]]]: """When values in amendments are given as lists, explode them to have all possible combinations of values.""" amendments_expanded = [] diff --git a/etl/steps/data/garden/ember/2023-07-10/shared.py b/etl/steps/data/garden/ember/2023-07-10/shared.py index bc7f5d72c21..5c8e8d173e0 100644 --- a/etl/steps/data/garden/ember/2023-07-10/shared.py +++ b/etl/steps/data/garden/ember/2023-07-10/shared.py @@ -298,7 +298,7 @@ def add_region_aggregates( def _expand_combinations_in_amendments( - amendments: List[Tuple[Dict[Any, Any], Dict[Any, Any]]] + amendments: List[Tuple[Dict[Any, Any], Dict[Any, Any]]], ) -> List[Tuple[Dict[Any, Any], Dict[Any, Any]]]: """When values in amendments are given as lists, explode them to have all possible combinations of values.""" amendments_expanded = [] diff --git a/etl/steps/data/garden/ember/2023-07-10/yearly_electricity.py b/etl/steps/data/garden/ember/2023-07-10/yearly_electricity.py index f249661ee1b..16e7b28e597 100644 --- a/etl/steps/data/garden/ember/2023-07-10/yearly_electricity.py +++ b/etl/steps/data/garden/ember/2023-07-10/yearly_electricity.py @@ -1,6 +1,4 @@ -"""Garden step for Ember's Yearly Electricity Data. - -""" +"""Garden step for Ember's Yearly Electricity Data.""" import numpy as np import owid.catalog.processing as pr diff --git a/etl/steps/data/garden/ember/2023-12-12/european_electricity_review.py b/etl/steps/data/garden/ember/2023-12-12/european_electricity_review.py index 29ac415248b..bdcabd97a80 100644 --- a/etl/steps/data/garden/ember/2023-12-12/european_electricity_review.py +++ b/etl/steps/data/garden/ember/2023-12-12/european_electricity_review.py @@ -1,6 +1,4 @@ -"""Garden step for European Electricity Review (Ember, 2022). - -""" +"""Garden step for European Electricity Review (Ember, 2022).""" from owid.catalog import Table from owid.datautils import dataframes diff --git a/etl/steps/data/garden/ember/2023-12-12/yearly_electricity.py b/etl/steps/data/garden/ember/2023-12-12/yearly_electricity.py index 23a46702346..80dbda93425 100644 --- a/etl/steps/data/garden/ember/2023-12-12/yearly_electricity.py +++ b/etl/steps/data/garden/ember/2023-12-12/yearly_electricity.py @@ -1,6 +1,4 @@ -"""Garden step for Ember's Yearly Electricity Data. - -""" +"""Garden step for Ember's Yearly Electricity Data.""" import owid.catalog.processing as pr from owid.catalog import Dataset, Table diff --git a/etl/steps/data/garden/ember/2024-05-08/yearly_electricity.py b/etl/steps/data/garden/ember/2024-05-08/yearly_electricity.py index d196ec6f757..30c26f56680 100644 --- a/etl/steps/data/garden/ember/2024-05-08/yearly_electricity.py +++ b/etl/steps/data/garden/ember/2024-05-08/yearly_electricity.py @@ -4,6 +4,7 @@ version, where data needed to be combined with the European Electricity Review). """ + from typing import Dict import owid.catalog.processing as pr diff --git a/etl/steps/data/garden/ember/2024-11-20/european_wholesale_electricity_prices.countries.json b/etl/steps/data/garden/ember/2024-11-20/european_wholesale_electricity_prices.countries.json new file mode 100644 index 00000000000..48aa385d8ee --- /dev/null +++ b/etl/steps/data/garden/ember/2024-11-20/european_wholesale_electricity_prices.countries.json @@ -0,0 +1,31 @@ +{ + "Austria": "Austria", + "Belgium": "Belgium", + "Bulgaria": "Bulgaria", + "Croatia": "Croatia", + "Czechia": "Czechia", + "Denmark": "Denmark", + "Estonia": "Estonia", + "Finland": "Finland", + "France": "France", + "Germany": "Germany", + "Greece": "Greece", + "Hungary": "Hungary", + "Ireland": "Ireland", + "Italy": "Italy", + "Latvia": "Latvia", + "Lithuania": "Lithuania", + "Luxembourg": "Luxembourg", + "Netherlands": "Netherlands", + "North Macedonia": "North Macedonia", + "Norway": "Norway", + "Poland": "Poland", + "Portugal": "Portugal", + "Romania": "Romania", + "Serbia": "Serbia", + "Slovakia": "Slovakia", + "Slovenia": "Slovenia", + "Spain": "Spain", + "Sweden": "Sweden", + "Switzerland": "Switzerland" +} diff --git a/etl/steps/data/garden/ember/2024-11-20/european_wholesale_electricity_prices.meta.yml b/etl/steps/data/garden/ember/2024-11-20/european_wholesale_electricity_prices.meta.yml new file mode 100644 index 00000000000..98ad5c38e4e --- /dev/null +++ b/etl/steps/data/garden/ember/2024-11-20/european_wholesale_electricity_prices.meta.yml @@ -0,0 +1,27 @@ +definitions: + common: + presentation: + topic_tags: + - Energy + processing_level: minor + +dataset: + update_period_days: 365 + +tables: + european_wholesale_electricity_prices_monthly: + variables: + price: + title: Electricity wholesale monthly price + unit: "current euros per megawatt-hour" + short_unit: "€/MWh" + description_short: |- + Monthly average day-ahead spot prices per [megawatt-hour](#dod:watt-hours) of electricity sold. + european_wholesale_electricity_prices_annual: + variables: + price: + title: Electricity wholesale annual price + unit: "current euros per megawatt-hour" + short_unit: "€/MWh" + description_short: |- + Annual average day-ahead spot prices per [megawatt-hour](#dod:watt-hours) of electricity sold. diff --git a/etl/steps/data/garden/ember/2024-11-20/european_wholesale_electricity_prices.py b/etl/steps/data/garden/ember/2024-11-20/european_wholesale_electricity_prices.py new file mode 100644 index 00000000000..e6a27829c94 --- /dev/null +++ b/etl/steps/data/garden/ember/2024-11-20/european_wholesale_electricity_prices.py @@ -0,0 +1,57 @@ +"""Load a meadow dataset and create a garden dataset.""" + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +# Select and rename columns. +COLUMNS = { + "country": "country", + "date": "date", + "price__eur_mwhe": "price", +} + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("european_wholesale_electricity_prices") + + # Read table from meadow dataset. + tb_monthly = ds_meadow.read("european_wholesale_electricity_prices") + + # + # Process data. + # + # Select and rename columns. + tb_monthly = tb_monthly[list(COLUMNS)].rename(columns=COLUMNS, errors="raise") + + # Harmonize country names. + tb_monthly = geo.harmonize_countries(df=tb_monthly, countries_file=paths.country_mapping_path) + + # Ember provides monthly data, so we can create a monthly table of wholesale electricity prices. + # But we also need to create an annual table of average wholesale electricity prices. + tb_annual = tb_monthly.copy() + tb_annual["year"] = tb_annual["date"].str[:4].astype("Int64") + # NOTE: We will include only complete years. This means that the latest year will not be included. But also, we will disregard country-years like Ireland 2022, which only has data for a few months, for some reason. + n_months = tb_annual.groupby(["country", "year"], observed=True, as_index=False)["date"].transform("count") + tb_annual = ( + tb_annual[n_months == 12].groupby(["country", "year"], observed=True, as_index=False).agg({"price": "mean"}) + ) + + # Improve table formats. + tb_monthly = tb_monthly.format(["country", "date"], short_name="european_wholesale_electricity_prices_monthly") + tb_annual = tb_annual.format(short_name="european_wholesale_electricity_prices_annual") + + # + # Save outputs. + # + # Create a new garden dataset. + ds_garden = create_dataset(dest_dir, tables=[tb_monthly, tb_annual], check_variables_metadata=True) + + # Save changes in the new garden dataset. + ds_garden.save() diff --git a/etl/steps/data/garden/emissions/2023-12-12/owid_co2.py b/etl/steps/data/garden/emissions/2023-12-12/owid_co2.py index d93cf411c0c..d186b77b9f8 100644 --- a/etl/steps/data/garden/emissions/2023-12-12/owid_co2.py +++ b/etl/steps/data/garden/emissions/2023-12-12/owid_co2.py @@ -11,7 +11,6 @@ """ - import numpy as np from owid.catalog import Dataset, Origin, Table diff --git a/etl/steps/data/garden/emissions/2024-04-08/national_contributions.py b/etl/steps/data/garden/emissions/2024-04-08/national_contributions.py index 548ea63802c..6309786685b 100644 --- a/etl/steps/data/garden/emissions/2024-04-08/national_contributions.py +++ b/etl/steps/data/garden/emissions/2024-04-08/national_contributions.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - import owid.catalog.processing as pr from owid.catalog import Dataset, Table, Variable from owid.datautils.dataframes import map_series diff --git a/etl/steps/data/garden/emissions/2024-11-21/national_contributions.py b/etl/steps/data/garden/emissions/2024-11-21/national_contributions.py index ebe2fb4ae8c..540aed18dec 100644 --- a/etl/steps/data/garden/emissions/2024-11-21/national_contributions.py +++ b/etl/steps/data/garden/emissions/2024-11-21/national_contributions.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - import owid.catalog.processing as pr from owid.catalog import Dataset, Table, Variable from owid.datautils.dataframes import map_series diff --git a/etl/steps/data/garden/energy/2023-07-10/energy_mix.py b/etl/steps/data/garden/energy/2023-07-10/energy_mix.py index e558257f487..fa52fc55002 100644 --- a/etl/steps/data/garden/energy/2023-07-10/energy_mix.py +++ b/etl/steps/data/garden/energy/2023-07-10/energy_mix.py @@ -1,6 +1,4 @@ -"""Generate the energy mix dataset using data from the Energy Institute Statistical Review of World Energy. - -""" +"""Generate the energy mix dataset using data from the Energy Institute Statistical Review of World Energy.""" import numpy as np from owid.catalog import Dataset, Table diff --git a/etl/steps/data/garden/energy/2023-07-10/fossil_fuel_reserves_production_ratio.py b/etl/steps/data/garden/energy/2023-07-10/fossil_fuel_reserves_production_ratio.py index 8c1d7e27674..3913989965f 100644 --- a/etl/steps/data/garden/energy/2023-07-10/fossil_fuel_reserves_production_ratio.py +++ b/etl/steps/data/garden/energy/2023-07-10/fossil_fuel_reserves_production_ratio.py @@ -1,6 +1,4 @@ -"""Load Statistical Review of World Energy and create variables of reserves-to-production ratios of fossil fuels. - -""" +"""Load Statistical Review of World Energy and create variables of reserves-to-production ratios of fossil fuels.""" from owid.catalog import Dataset, Table diff --git a/etl/steps/data/garden/energy/2023-12-12/energy_mix.py b/etl/steps/data/garden/energy/2023-12-12/energy_mix.py index 4da60f0ecf2..9589ec608fe 100644 --- a/etl/steps/data/garden/energy/2023-12-12/energy_mix.py +++ b/etl/steps/data/garden/energy/2023-12-12/energy_mix.py @@ -1,6 +1,4 @@ -"""Generate the energy mix dataset using data from the Energy Institute Statistical Review of World Energy. - -""" +"""Generate the energy mix dataset using data from the Energy Institute Statistical Review of World Energy.""" import numpy as np from owid.catalog import Dataset, Table diff --git a/etl/steps/data/garden/energy/2023-12-12/fossil_fuel_reserves_production_ratio.py b/etl/steps/data/garden/energy/2023-12-12/fossil_fuel_reserves_production_ratio.py index dea19c03ae6..64fe0e404eb 100644 --- a/etl/steps/data/garden/energy/2023-12-12/fossil_fuel_reserves_production_ratio.py +++ b/etl/steps/data/garden/energy/2023-12-12/fossil_fuel_reserves_production_ratio.py @@ -1,6 +1,4 @@ -"""Load Statistical Review of World Energy and create variables of reserves-to-production ratios of fossil fuels. - -""" +"""Load Statistical Review of World Energy and create variables of reserves-to-production ratios of fossil fuels.""" from owid.catalog import Table diff --git a/etl/steps/data/garden/energy/2024-05-08/energy_mix.py b/etl/steps/data/garden/energy/2024-05-08/energy_mix.py index 7dbcd3e3651..4523375346a 100644 --- a/etl/steps/data/garden/energy/2024-05-08/energy_mix.py +++ b/etl/steps/data/garden/energy/2024-05-08/energy_mix.py @@ -1,6 +1,4 @@ -"""Generate the energy mix dataset using data from the Energy Institute Statistical Review of World Energy. - -""" +"""Generate the energy mix dataset using data from the Energy Institute Statistical Review of World Energy.""" import numpy as np from owid.catalog import Dataset, Table diff --git a/etl/steps/data/garden/energy/2024-05-08/fossil_fuel_reserves_production_ratio.py b/etl/steps/data/garden/energy/2024-05-08/fossil_fuel_reserves_production_ratio.py index 28d708eaa4f..2d6ab167e79 100644 --- a/etl/steps/data/garden/energy/2024-05-08/fossil_fuel_reserves_production_ratio.py +++ b/etl/steps/data/garden/energy/2024-05-08/fossil_fuel_reserves_production_ratio.py @@ -1,6 +1,4 @@ -"""Load Statistical Review of World Energy and create variables of reserves-to-production ratios of fossil fuels. - -""" +"""Load Statistical Review of World Energy and create variables of reserves-to-production ratios of fossil fuels.""" from owid.catalog import Table diff --git a/etl/steps/data/garden/energy/2024-06-20/energy_mix.py b/etl/steps/data/garden/energy/2024-06-20/energy_mix.py index 7dbcd3e3651..4523375346a 100644 --- a/etl/steps/data/garden/energy/2024-06-20/energy_mix.py +++ b/etl/steps/data/garden/energy/2024-06-20/energy_mix.py @@ -1,6 +1,4 @@ -"""Generate the energy mix dataset using data from the Energy Institute Statistical Review of World Energy. - -""" +"""Generate the energy mix dataset using data from the Energy Institute Statistical Review of World Energy.""" import numpy as np from owid.catalog import Dataset, Table diff --git a/etl/steps/data/garden/energy/2024-06-20/fossil_fuel_reserves_production_ratio.py b/etl/steps/data/garden/energy/2024-06-20/fossil_fuel_reserves_production_ratio.py index 28d708eaa4f..2d6ab167e79 100644 --- a/etl/steps/data/garden/energy/2024-06-20/fossil_fuel_reserves_production_ratio.py +++ b/etl/steps/data/garden/energy/2024-06-20/fossil_fuel_reserves_production_ratio.py @@ -1,6 +1,4 @@ -"""Load Statistical Review of World Energy and create variables of reserves-to-production ratios of fossil fuels. - -""" +"""Load Statistical Review of World Energy and create variables of reserves-to-production ratios of fossil fuels.""" from owid.catalog import Table diff --git a/etl/steps/data/garden/energy/2024-11-20/energy_prices.meta.yml b/etl/steps/data/garden/energy/2024-11-20/energy_prices.meta.yml new file mode 100644 index 00000000000..6a934bc8bd5 --- /dev/null +++ b/etl/steps/data/garden/energy/2024-11-20/energy_prices.meta.yml @@ -0,0 +1,11 @@ +definitions: + common: + processing_level: major + presentation: + topic_tags: + - Energy + +dataset: + title: European Energy Prices + update_period_days: 365 + diff --git a/etl/steps/data/garden/energy/2024-11-20/energy_prices.py b/etl/steps/data/garden/energy/2024-11-20/energy_prices.py new file mode 100644 index 00000000000..73621d987ad --- /dev/null +++ b/etl/steps/data/garden/energy/2024-11-20/energy_prices.py @@ -0,0 +1,60 @@ +"""Compilation of energy prices datasets.""" + +import owid.catalog.processing as pr + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current data step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load data. + # + # Load Eurostat data on European gas and electricity prices. + ds_eurostat = paths.load_dataset("gas_and_electricity_prices") + tb_eurostat_euro = ds_eurostat.read("gas_and_electricity_price_components_euro_flat") + tb_eurostat_pps = ds_eurostat.read("gas_and_electricity_price_components_pps_flat") + + # Load Ember data on European wholesale electricity prices. + ds_ember = paths.load_dataset("european_wholesale_electricity_prices") + tb_ember_monthly = ds_ember.read("european_wholesale_electricity_prices_monthly") + tb_ember_annual = ds_ember.read("european_wholesale_electricity_prices_annual") + + # + # Process data. + # + # Rename columns in all tables to have consistent dimensions. + tb_eurostat_euro = tb_eurostat_euro.rename( + columns={ + column: f"annual_{column}" for column in tb_eurostat_euro.columns if column not in ["country", "year"] + }, + errors="raise", + ) + tb_eurostat_pps = tb_eurostat_pps.rename( + columns={column: f"annual_{column}" for column in tb_eurostat_pps.columns if column not in ["country", "year"]}, + errors="raise", + ) + tb_ember_monthly = tb_ember_monthly.rename( + columns={"price": "monthly_electricity_all_wholesale_euro"}, errors="raise" + ) + tb_ember_annual = tb_ember_annual.rename(columns={"price": "annual_electricity_all_wholesale_euro"}, errors="raise") + + # Create a combined annual table. + tb_annual = pr.multi_merge( + tables=[tb_eurostat_euro, tb_eurostat_pps, tb_ember_annual], on=["country", "year"], how="outer" + ) + tb_annual = tb_annual.format(short_name="energy_prices_annual") + + # Create a combined monthly table. + # For now, only Ember has monthly data. + tb_monthly = tb_ember_monthly.copy() + tb_monthly = tb_monthly.format(keys=["country", "date"], short_name="energy_prices_monthly") + + # + # Save outputs. + # + # Create a new dataset with the same metadata as meadow + ds_garden = create_dataset(dest_dir=dest_dir, tables=[tb_annual, tb_monthly], check_variables_metadata=True) + ds_garden.save() diff --git a/etl/steps/data/garden/ess/2023-08-02/ess_trust.py b/etl/steps/data/garden/ess/2023-08-02/ess_trust.py index 7148c578b38..f382eae78e0 100644 --- a/etl/steps/data/garden/ess/2023-08-02/ess_trust.py +++ b/etl/steps/data/garden/ess/2023-08-02/ess_trust.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - from owid.catalog import Table from etl.data_helpers import geo diff --git a/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.countries.json b/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.countries.json new file mode 100644 index 00000000000..f11ceb2730a --- /dev/null +++ b/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.countries.json @@ -0,0 +1,44 @@ +{ + "BE": "Belgium", + "BG": "Bulgaria", + "CZ": "Czechia", + "DK": "Denmark", + "DE": "Germany", + "EE": "Estonia", + "IE": "Ireland", + "EL": "Greece", + "ES": "Spain", + "FR": "France", + "HR": "Croatia", + "IT": "Italy", + "CY": "Cyprus", + "LV": "Latvia", + "LT": "Lithuania", + "LU": "Luxembourg", + "HU": "Hungary", + "MT": "Malta", + "NL": "Netherlands", + "AT": "Austria", + "PL": "Poland", + "PT": "Portugal", + "RO": "Romania", + "SI": "Slovenia", + "SK": "Slovakia", + "FI": "Finland", + "SE": "Sweden", + "IS": "Iceland", + "LI": "Liechtenstein", + "NO": "Norway", + "BA": "Bosnia and Herzegovina", + "ME": "Montenegro", + "MD": "Moldova", + "GE": "Georgia", + "MK": "North Macedonia", + "AL": "Albania", + "RS": "Serbia", + "TR": "Turkey", + "XK": "Kosovo", + "EU27_2020": "European Union (27)", + "UA": "Ukraine", + "UK": "United Kingdom" +} diff --git a/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.excluded_countries.json b/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.excluded_countries.json new file mode 100644 index 00000000000..7745db9ff44 --- /dev/null +++ b/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.excluded_countries.json @@ -0,0 +1 @@ +["EA"] diff --git a/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.meta.yml b/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.meta.yml new file mode 100644 index 00000000000..5b27ffbfd55 --- /dev/null +++ b/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.meta.yml @@ -0,0 +1,1334 @@ +definitions: + common: + presentation: + topic_tags: + - Energy + processing_level: major + + # Definitions of consumer type: + description_key_household_consumer: |- + A household consumer of gas or electricity is a residential user who consumes energy for domestic purposes like heating, cooking, lighting, and appliances. + description_key_non_household_consumer: |- + A non-household consumer of gas or electricity is a business or organization using energy for purposes like industry, services, offices, or agriculture. + # Definitions of consumer bands: + description_key_all_consumer_bands: |- + Prices are a weighted average across all consumption bands, from low to high energy consumers, based on the relative consumption shares reported by each country. + # Definitions of price components: + # Eurostat's definitions can be found: + # * For gas: https://ec.europa.eu/eurostat/cache/metadata/en/nrg_pc_202_sims.htm + # * For electricity: https://ec.europa.eu/eurostat/cache/metadata/en/nrg_pc_204_sims.htm + # * Gas - Energy and supply: commodity price for natural gas paid by the supplier or the price of natural gas at the point of entry into the transmission system, including, if applicable, the following end-user costs: storage costs plus costs relating to the sale of natural gas to final customers. + # * Electricity - Energy and supply: generation, aggregation, balancing energy, supplied energy costs, customer services, after-sales management and other supply costs. + # * Gas/electricity - Network cost: transmission and distribution tariffs, transmission and distribution losses, network costs, after-sale service costs, system service costs, and meter rental and metering costs. + # * Gas/electricity - Value added taxes (VAT): as defined in Council Directive 2006/112/EC. + # * Gas/electricity - Renewable taxes: taxes, fees, levies or charges relating to the promotion of renewable energy sources, energy efficiency and CHP generation. + # * Gas - Capacity taxes: taxes, fees, levies or charges relating to strategic stockpiles, capacity payments and energy security; taxes on natural gas distribution; stranded costs and levies on financing energy regulatory authorities or market and system operators. + # * Electricity - Capacity taxes: Taxes, fees, levies or charges relating to capacity payments, energy security and generation adequacy; taxes on coal industry restructuring; taxes on electricity distribution; stranded costs and levies on financing energy regulatory authorities or market and system operators. + # * Gas/electricity - Environmental taxes: taxes, fees, levies or charges relating to air quality and for other environmental purposes; taxes on emissions of CO2 or other greenhouse gases. This component includes the excise duties. + # * Gas/electricity - All other taxes: taxes, fees, levies or charges not covered by any of the previous four categories: support for district heating; local or regional fiscal charges; island compensation; concession fees relating to licences and fees for the occupation of land and public or private property by networks or other devices. + # * Electricity - Nuclear taxes: taxes, fees, levies or charges relating to the nuclear sector, including nuclear decommissioning, inspections and fees for nuclear installations. + description_key_gas_component_energy_and_supply: |- + "Energy and supply" is the cost of natural gas, including the commodity price paid by the supplier or the price at the point of entry into the transmission system. It also includes, where applicable, end-user costs such as storage and expenses related to the sale of gas to final customers. + description_key_gas_component_network_cost: &description_key_gas_component_network_cost |- + "Network costs" include the tariffs for transmission and distribution, costs from transmission and distribution losses, and system-related expenses such as after-sale services, system services, meter rental, and metering. + description_key_gas_component_taxes_fees_levies_and_charges: &description_key_gas_component_taxes_fees_levies_and_charges |- + "Taxes, fees, levies, and charges" include value-added tax (VAT), renewable taxes, capacity taxes, environmental taxes, and all other taxes, fees, levies, or charges. + description_key_gas_component_vat: &description_key_gas_component_vat |- + "Value-added tax (VAT)" is a consumption tax applied to the final price of gas or electricity, as defined in Council Directive 2006/112/EC. + description_key_gas_component_renewable_taxes: &description_key_gas_component_renewable_taxes |- + "Renewable taxes" are taxes, fees, levies, or charges that promote renewable energy sources, energy efficiency, and combined heat and power (CHP) generation. + description_key_gas_component_capacity_taxes: |- + "Capacity taxes" are charges related to maintaining energy security and the gas distribution system. They include taxes or fees for strategic gas stockpiles, payments to ensure sufficient supply capacity, costs for supporting energy regulatory authorities or system operators, and levies to cover past infrastructure investments (stranded costs). + description_key_gas_component_environmental_taxes: &description_key_gas_component_environmental_taxes |- + "Environmental taxes" are charges aimed at addressing environmental concerns, such as improving air quality and reducing greenhouse gas emissions (e.g., CO₂ taxes). This category also includes excise duties—specific taxes applied to goods or activities like fuels and energy use, which contribute to environmental impacts. + description_key_gas_component_other_taxes: &description_key_gas_component_other_taxes |- + "All other taxes" include fees, levies, or charges not covered by other tax components. They may relate to district heating support, local or regional fiscal charges, island compensation, concession fees for licenses, or fees for occupying land or property with networks or other devices. + description_key_gas_component_total_price_including_taxes: &description_key_gas_component_total_price_including_taxes |- + "Total price including taxes" is the sum of all price components, including energy and supply, network costs, taxes, fees, levies and charges. + description_key_electricity_component_energy_and_supply: |- + "Energy and supply" is the cost of electricity generation, aggregation, balancing, and other supply-related activities, including end-user costs such as customer services and after-sales management. + description_key_electricity_component_network_cost: *description_key_gas_component_network_cost + description_key_electricity_component_vat: *description_key_gas_component_vat + description_key_electricity_component_renewable_taxes: *description_key_gas_component_renewable_taxes + description_key_electricity_component_capacity_taxes: |- + "Capacity taxes" are charges aimed at ensuring energy security and sufficient electricity supply. They include fees for maintaining generation capacity, supporting energy regulatory authorities or system operators, restructuring the coal industry, and improving electricity distribution. They may also cover costs from past infrastructure investments (stranded costs). + description_key_electricity_component_environmental_taxes: *description_key_gas_component_environmental_taxes + description_key_electricity_component_other_taxes: *description_key_gas_component_other_taxes + description_key_electricity_component_nuclear_taxes: |- + "Nuclear taxes" are charges related to the nuclear sector, including decommissioning, inspections, and fees for nuclear installations. + description_key_electricity_component_total_price_including_taxes: *description_key_gas_component_total_price_including_taxes + description_key_electricity_component_taxes_fees_levies_and_charges: *description_key_gas_component_taxes_fees_levies_and_charges + # Definitions of price levels: + # Original Eurostat definitions of price levels: + # Level 1 prices ("X_TAX"): Prices excluding taxes and levies. + # Level 2 prices ("X_VAT"): Prices excluding VAT and other recoverable taxes and levies. + # Level 3 prices ("I_TAX"): Prices including all taxes and levies. + description_key_price_excluding_taxes: |- + Prices represent the base cost of energy, excluding all taxes, levies, and VAT. These prices reflect the pure costs of energy and supply, network, and other market-related services. + description_key_price_excluding_vat: |- + Prices include the base cost of energy and non-recoverable taxes and levies. They exclude VAT and other recoverable taxes. + description_key_price_including_all_taxes: |- + Prices represent the total price paid by end consumers, including all taxes, levies, and VAT. + + +dataset: + update_period_days: 365 + + +tables: + gas_and_electricity_prices: + variables: + price_euro: + title: Price + unit: euro + short_unit: "€" + description_short: Energy price in euro. + price_pps: + title: Price (purchasing power) + unit: purchasing power standard + short_unit: PPS + description_short: Energy price in purchasing power standard. + gas_and_electricity_price_components_euro_flat: + title: Gas and electricity price components in Europe + common: + description_short: Price components are given in euros per [megawatt-hour](#dod:watt-hours). They are not adjusted for inflation or differences in living costs between countries. + unit: 'current euros per megawatt-hour' + short_unit: "€/MWh" + variables: + electricity_household_capacity_taxes_euro: + title: Electricity price component (euros) for household consumers - Capacity taxes + display: + name: Capacity taxes + presentation: + title_public: Electricity price component for household consumers - Capacity taxes + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_capacity_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_capacity_taxes_euro: + title: Electricity price component (euros) for non-household consumers - Capacity taxes + display: + name: Capacity taxes + presentation: + title_public: Electricity price component for non-household consumers - Capacity taxes + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_capacity_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_capacity_taxes_allowances_euro: + title: Electricity price component (euros) for household consumers - Capacity taxes allowances + display: + name: Capacity taxes allowances + presentation: + title_public: Electricity price component for household consumers - Capacity taxes allowances + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_capacity_taxes_allowances}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_energy_and_supply_euro: + title: Electricity price component (euros) for household consumers - Energy and supply + display: + name: Energy and supply + presentation: + title_public: Electricity price component for household consumers - Energy and supply + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_energy_and_supply}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_energy_and_supply_euro: + title: Electricity price component (euros) for non-household consumers - Energy and supply + display: + name: Energy and supply + presentation: + title_public: Electricity price component for non-household consumers - Energy and supply + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_energy_and_supply}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_environmental_taxes_euro: + title: Electricity price component (euros) for household consumers - Environmental taxes + display: + name: Environmental taxes + presentation: + title_public: Electricity price component for household consumers - Environmental taxes + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_environmental_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_environmental_taxes_euro: + title: Electricity price component (euros) for non-household consumers - Environmental taxes + display: + name: Environmental taxes + presentation: + title_public: Electricity price component for non-household consumers - Environmental taxes + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_environmental_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_environmental_taxes_allowance_euro: + title: Electricity price component (euros) for household consumers - Environmental taxes allowance + display: + name: Environmental taxes allowance + presentation: + title_public: Electricity price component for household consumers - Environmental taxes allowance + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_environmental_taxes_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_network_costs_euro: + title: Electricity price component (euros) for household consumers - Network costs + display: + name: Network costs + presentation: + title_public: Electricity price component for household consumers - Network costs + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_network_cost}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_network_costs_euro: + title: Electricity price component (euros) for non-household consumers - Network costs + display: + name: Network costs + presentation: + title_public: Electricity price component for non-household consumers - Network costs + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_network_cost}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_nuclear_taxes_euro: + title: Electricity price component (euros) for household consumers - Nuclear taxes + display: + name: Nuclear taxes + presentation: + title_public: Electricity price component for household consumers - Nuclear taxes + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_nuclear_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_nuclear_taxes_euro: + title: Electricity price component (euros) for non-household consumers - Nuclear taxes + display: + name: Nuclear taxes + presentation: + title_public: Electricity price component for non-household consumers - Nuclear taxes + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_nuclear_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_nuclear_taxes_allowance_euro: + title: Electricity price component (euros) for household consumers - Nuclear taxes allowance + display: + name: Nuclear taxes allowance + presentation: + title_public: Electricity price component for household consumers - Nuclear taxes allowance + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_nuclear_taxes_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_other_euro: + title: Electricity price component (euros) for household consumers - Other + display: + name: Other + presentation: + title_public: Electricity price component for household consumers - Other + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_other_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_other_euro: + title: Electricity price component (euros) for non-household consumers - Other + display: + name: Other + presentation: + title_public: Electricity price component for non-household consumers - Other + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_other_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_other_allowance_euro: + title: Electricity price component (euros) for household consumers - Other allowance + display: + name: Other allowance + presentation: + title_public: Electricity price component for household consumers - Other allowance + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_other_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_renewable_taxes_euro: + title: Electricity price component (euros) for household consumers - Renewable taxes + display: + name: Renewable taxes + presentation: + title_public: Electricity price component for household consumers - Renewable taxes + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_renewable_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_renewable_taxes_euro: + title: Electricity price component (euros) for non-household consumers - Renewable taxes + display: + name: Renewable taxes + presentation: + title_public: Electricity price component for non-household consumers - Renewable taxes + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_renewable_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_renewable_taxes_allowance_euro: + title: Electricity price component (euros) for household consumers - Renewable taxes allowance + display: + name: Renewable taxes allowance + presentation: + title_public: Electricity price component for household consumers - Renewable taxes allowance + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_renewable_taxes_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_taxes_fees_levies_and_charges_euro: + title: Electricity price component (euros) for household consumers - Taxes, fees, levies, and charges + display: + name: Taxes, fees, levies, and charges + presentation: + title_public: Electricity price component for household consumers - Taxes, fees, levies, and charges + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_taxes_fees_levies_and_charges}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_taxes_fees_levies_and_charges_euro: + title: Electricity price component (euros) for non-household consumers - Taxes, fees, levies, and charges + display: + name: Taxes, fees, levies, and charges + presentation: + title_public: Electricity price component for non-household consumers - Taxes, fees, levies, and charges + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_taxes_fees_levies_and_charges}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_value_added_tax_vat_euro: + title: Electricity price component (euros) for household consumers - Value added tax (VAT) + display: + name: Value added tax (VAT) + presentation: + title_public: Electricity price component for household consumers - Value added tax (VAT) + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_vat}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_value_added_tax_vat_euro: + title: Electricity price component (euros) for non-household consumers - Value added tax (VAT) + display: + name: Value added tax (VAT) + presentation: + title_public: Electricity price component for non-household consumers - Value added tax (VAT) + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_vat}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_capacity_taxes_euro: + title: Gas price component (euros) for household consumers - Capacity taxes + display: + name: Capacity taxes + presentation: + title_public: Gas price component for household consumers - Capacity taxes + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_capacity_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_energy_and_supply_euro: + title: Gas price component (euros) for household consumers - Energy and supply + display: + name: Energy and supply + presentation: + title_public: Gas price component for household consumers - Energy and supply + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_energy_and_supply}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_environmental_taxes_euro: + title: Gas price component (euros) for household consumers - Environmental taxes + display: + name: Environmental taxes + presentation: + title_public: Gas price component for household consumers - Environmental taxes + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_environmental_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_network_costs_euro: + title: Gas price component (euros) for household consumers - Network costs + display: + name: Network costs + presentation: + title_public: Gas price component for household consumers - Network costs + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_network_cost}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_other_euro: + title: Gas price component (euros) for household consumers - Other + display: + name: Other + presentation: + title_public: Gas price component for household consumers - Other + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_other_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_renewable_taxes_euro: + title: Gas price component (euros) for household consumers - Renewable taxes + display: + name: Renewable taxes + presentation: + title_public: Gas price component for household consumers - Renewable taxes + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_renewable_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_taxes_fees_levies_and_charges_euro: + title: Gas price component (euros) for household consumers - Taxes, fees, levies, and charges + display: + name: Taxes, fees, levies, and charges + presentation: + title_public: Gas price component for household consumers - Taxes, fees, levies, and charges + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_taxes_fees_levies_and_charges}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_value_added_tax_vat_euro: + title: Gas price component (euros) for household consumers - Value added tax (VAT) + display: + name: Value added tax (VAT) + presentation: + title_public: Gas price component for household consumers - Value added tax (VAT) + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_vat}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_capacity_taxes_euro: + title: Gas price component (euros) for non-household consumers - Capacity taxes + display: + name: Capacity taxes + presentation: + title_public: Gas price component for non-household consumers - Capacity taxes + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_capacity_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_energy_and_supply_euro: + title: Gas price component (euros) for non-household consumers - Energy and supply + display: + name: Energy and supply + presentation: + title_public: Gas price component for non-household consumers - Energy and supply + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_energy_and_supply}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_environmental_taxes_euro: + title: Gas price component (euros) for non-household consumers - Environmental taxes + display: + name: Environmental taxes + presentation: + title_public: Gas price component for non-household consumers - Environmental taxes + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_environmental_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_network_costs_euro: + title: Gas price component (euros) for non-household consumers - Network costs + display: + name: Network costs + presentation: + title_public: Gas price component for non-household consumers - Network costs + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_network_cost}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_other_euro: + title: Gas price component (euros) for non-household consumers - Other + display: + name: Other + presentation: + title_public: Gas price component for non-household consumers - Other + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_other_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_renewable_taxes_euro: + title: Gas price component (euros) for non-household consumers - Renewable taxes + display: + name: Renewable taxes + presentation: + title_public: Gas price component for non-household consumers - Renewable taxes + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_renewable_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_taxes_fees_levies_and_charges_euro: + title: Gas price component (euros) for non-household consumers - Taxes, fees, levies, and charges + display: + name: Taxes, fees, levies, and charges + presentation: + title_public: Gas price component for non-household consumers - Taxes, fees, levies, and charges + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_taxes_fees_levies_and_charges}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_value_added_tax_vat_euro: + title: Gas price component (euros) for non-household consumers - Value added tax (VAT) + display: + name: Value added tax (VAT) + presentation: + title_public: Gas price component for non-household consumers - Value added tax (VAT) + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_vat}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_taxes_fees_levies_and_charges_allowance_euro: + title: Electricity price component (euros) for household consumers - Taxes, fees, levies, and charges allowance + display: + name: Taxes, fees, levies, and charges allowance + presentation: + title_public: Electricity price component for household consumers - Taxes, fees, levies, and charges allowance + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_taxes_fees_levies_and_charges_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_total_price_including_taxes_euro: + title: Electricity price component (euros) for household consumers - Total price including taxes + display: + name: Total price including taxes + presentation: + title_public: Electricity price component for household consumers - Total price including taxes + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_total_price_including_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_total_price_including_taxes_euro: + title: Electricity price component (euros) for non-household consumers - Total price including taxes + display: + name: Total price including taxes + presentation: + title_public: Electricity price component for non-household consumers - Total price including taxes + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_total_price_including_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_total_price_including_taxes_euro: + title: Gas price component (euros) for household consumers - Total price including taxes + display: + name: Total price including taxes + presentation: + title_public: Gas price component for household consumers - Total price including taxes + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_total_price_including_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_total_price_including_taxes_euro: + title: Gas price component (euros) for non-household consumers - Total price including taxes + display: + name: Total price including taxes + presentation: + title_public: Gas price component for non-household consumers - Total price including taxes + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_total_price_including_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_and_electricity_price_components_pps_flat: + title: Gas and electricity price components in Europe (PPS) + common: + description_short: Price components are given in purchasing power standard (PPS) per [megawatt-hour](#dod:watt-hours). + unit: "purchasing power standard per megawatt-hour" + short_unit: "PPS/MWh" + variables: + electricity_household_capacity_taxes_pps: + title: Electricity price component (PPS) for household consumers - Capacity taxes + display: + name: Capacity taxes + presentation: + title_public: Electricity price component (PPS) for household consumers - Capacity taxes + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_capacity_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_capacity_taxes_pps: + title: Electricity price component (PPS) for non-household consumers - Capacity taxes + display: + name: Capacity taxes + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Capacity taxes + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_capacity_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_capacity_taxes_allowances_pps: + title: Electricity price component (PPS) for household consumers - Capacity taxes allowances + display: + name: Capacity taxes allowances + presentation: + title_public: Electricity price component (PPS) for household consumers - Capacity taxes allowances + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_capacity_taxes_allowances}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_energy_and_supply_pps: + title: Electricity price component (PPS) for household consumers - Energy and supply + display: + name: Energy and supply + presentation: + title_public: Electricity price component (PPS) for household consumers - Energy and supply + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_energy_and_supply}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_energy_and_supply_pps: + title: Electricity price component (PPS) for non-household consumers - Energy and supply + display: + name: Energy and supply + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Energy and supply + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_energy_and_supply}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_environmental_taxes_pps: + title: Electricity price component (PPS) for household consumers - Environmental taxes + display: + name: Environmental taxes + presentation: + title_public: Electricity price component (PPS) for household consumers - Environmental taxes + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_environmental_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_environmental_taxes_pps: + title: Electricity price component (PPS) for non-household consumers - Environmental taxes + display: + name: Environmental taxes + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Environmental taxes + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_environmental_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_environmental_taxes_allowance_pps: + title: Electricity price component (PPS) for household consumers - Environmental taxes allowance + display: + name: Environmental taxes allowance + presentation: + title_public: Electricity price component (PPS) for household consumers - Environmental taxes allowance + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_environmental_taxes_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_network_costs_pps: + title: Electricity price component (PPS) for household consumers - Network costs + display: + name: Network costs + presentation: + title_public: Electricity price component (PPS) for household consumers - Network costs + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_network_cost}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_network_costs_pps: + title: Electricity price component (PPS) for non-household consumers - Network costs + display: + name: Network costs + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Network costs + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_network_cost}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_nuclear_taxes_pps: + title: Electricity price component (PPS) for household consumers - Nuclear taxes + display: + name: Nuclear taxes + presentation: + title_public: Electricity price component (PPS) for household consumers - Nuclear taxes + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_nuclear_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_nuclear_taxes_pps: + title: Electricity price component (PPS) for non-household consumers - Nuclear taxes + display: + name: Nuclear taxes + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Nuclear taxes + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_nuclear_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_nuclear_taxes_allowance_pps: + title: Electricity price component (PPS) for household consumers - Nuclear taxes allowance + display: + name: Nuclear taxes allowance + presentation: + title_public: Electricity price component (PPS) for household consumers - Nuclear taxes allowance + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_nuclear_taxes_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_other_pps: + title: Electricity price component (PPS) for household consumers - Other + display: + name: Other + presentation: + title_public: Electricity price component (PPS) for household consumers - Other + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_other_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_other_pps: + title: Electricity price component (PPS) for non-household consumers - Other + display: + name: Other + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Other + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_other_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_other_allowance_pps: + title: Electricity price component (PPS) for household consumers - Other allowance + display: + name: Other allowance + presentation: + title_public: Electricity price component (PPS) for household consumers - Other allowance + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_other_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_renewable_taxes_pps: + title: Electricity price component (PPS) for household consumers - Renewable taxes + display: + name: Renewable taxes + presentation: + title_public: Electricity price component (PPS) for household consumers - Renewable taxes + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_renewable_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_renewable_taxes_pps: + title: Electricity price component (PPS) for non-household consumers - Renewable taxes + display: + name: Renewable taxes + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Renewable taxes + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_renewable_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_renewable_taxes_allowance_pps: + title: Electricity price component (PPS) for household consumers - Renewable taxes allowance + display: + name: Renewable taxes allowance + presentation: + title_public: Electricity price component (PPS) for household consumers - Renewable taxes allowance + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_renewable_taxes_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_taxes_fees_levies_and_charges_pps: + title: Electricity price component (PPS) for household consumers - Taxes, fees, levies, and charges + display: + name: Taxes, fees, levies, and charges + presentation: + title_public: Electricity price component (PPS) for household consumers - Taxes, fees, levies, and charges + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_taxes_fees_levies_and_charges}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_taxes_fees_levies_and_charges_pps: + title: Electricity price component (PPS) for non-household consumers - Taxes, fees, levies, and charges + display: + name: Taxes, fees, levies, and charges + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Taxes, fees, levies, and charges + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_taxes_fees_levies_and_charges}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_value_added_tax_vat_pps: + title: Electricity price component (PPS) for household consumers - Value added tax (VAT) + display: + name: Value added tax (VAT) + presentation: + title_public: Electricity price component (PPS) for household consumers - Value added tax (VAT) + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_vat}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_value_added_tax_vat_pps: + title: Electricity price component (PPS) for non-household consumers - Value added tax (VAT) + display: + name: Value added tax (VAT) + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Value added tax (VAT) + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_vat}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_capacity_taxes_pps: + title: Gas price component (PPS) for household consumers - Capacity taxes + display: + name: Capacity taxes + presentation: + title_public: Gas price component (PPS) for household consumers - Capacity taxes + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_capacity_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_energy_and_supply_pps: + title: Gas price component (PPS) for household consumers - Energy and supply + display: + name: Energy and supply + presentation: + title_public: Gas price component (PPS) for household consumers - Energy and supply + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_energy_and_supply}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_environmental_taxes_pps: + title: Gas price component (PPS) for household consumers - Environmental taxes + display: + name: Environmental taxes + presentation: + title_public: Gas price component (PPS) for household consumers - Environmental taxes + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_environmental_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_network_costs_pps: + title: Gas price component (PPS) for household consumers - Network costs + display: + name: Network costs + presentation: + title_public: Gas price component (PPS) for household consumers - Network costs + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_network_cost}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_other_pps: + title: Gas price component (PPS) for household consumers - Other + display: + name: Other + presentation: + title_public: Gas price component (PPS) for household consumers - Other + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_other_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_renewable_taxes_pps: + title: Gas price component (PPS) for household consumers - Renewable taxes + display: + name: Renewable taxes + presentation: + title_public: Gas price component (PPS) for household consumers - Renewable taxes + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_renewable_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_taxes_fees_levies_and_charges_pps: + title: Gas price component (PPS) for household consumers - Taxes, fees, levies, and charges + display: + name: Taxes, fees, levies, and charges + presentation: + title_public: Gas price component (PPS) for household consumers - Taxes, fees, levies, and charges + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_taxes_fees_levies_and_charges}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_value_added_tax_vat_pps: + title: Gas price component (PPS) for household consumers - Value added tax (VAT) + display: + name: Value added tax (VAT) + presentation: + title_public: Gas price component (PPS) for household consumers - Value added tax (VAT) + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_vat}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_capacity_taxes_pps: + title: Gas price component (PPS) for non-household consumers - Capacity taxes + display: + name: Capacity taxes + presentation: + title_public: Gas price component (PPS) for non-household consumers - Capacity taxes + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_capacity_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_energy_and_supply_pps: + title: Gas price component (PPS) for non-household consumers - Energy and supply + display: + name: Energy and supply + presentation: + title_public: Gas price component (PPS) for non-household consumers - Energy and supply + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_energy_and_supply}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_environmental_taxes_pps: + title: Gas price component (PPS) for non-household consumers - Environmental taxes + display: + name: Environmental taxes + presentation: + title_public: Gas price component (PPS) for non-household consumers - Environmental taxes + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_environmental_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_network_costs_pps: + title: Gas price component (PPS) for non-household consumers - Network costs + display: + name: Network costs + presentation: + title_public: Gas price component (PPS) for non-household consumers - Network costs + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_network_cost}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_other_pps: + title: Gas price component (PPS) for non-household consumers - Other + display: + name: Other + presentation: + title_public: Gas price component (PPS) for non-household consumers - Other + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_other_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_renewable_taxes_pps: + title: Gas price component (PPS) for non-household consumers - Renewable taxes + display: + name: Renewable taxes + presentation: + title_public: Gas price component (PPS) for non-household consumers - Renewable taxes + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_renewable_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_taxes_fees_levies_and_charges_pps: + title: Gas price component (PPS) for non-household consumers - Taxes, fees, levies, and charges + display: + name: Taxes, fees, levies, and charges + presentation: + title_public: Gas price component (PPS) for non-household consumers - Taxes, fees, levies, and charges + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_taxes_fees_levies_and_charges}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_value_added_tax_vat_pps: + title: Gas price component (PPS) for non-household consumers - Value added tax (VAT) + display: + name: Value added tax (VAT) + presentation: + title_public: Gas price component (PPS) for non-household consumers - Value added tax (VAT) + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_vat}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_taxes_fees_levies_and_charges_allowance_pps: + title: Electricity price component (PPS) for household consumers - Taxes, fees, levies, and charges allowance + display: + name: Taxes, fees, levies, and charges allowance + presentation: + title_public: Electricity price component (PPS) for household consumers - Taxes, fees, levies, and charges allowance + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_taxes_fees_levies_and_charges_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_total_price_including_taxes_pps: + title: Electricity price component (PPS) for household consumers - Total price including taxes + display: + name: Total price including taxes + presentation: + title_public: Electricity price component (PPS) for household consumers - Total price including taxes + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_total_price_including_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_total_price_including_taxes_pps: + title: Electricity price component (PPS) for non-household consumers - Total price including taxes + display: + name: Total price including taxes + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Total price including taxes + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_total_price_including_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_total_price_including_taxes_pps: + title: Gas price component (PPS) for household consumers - Total price including taxes + display: + name: Total price including taxes + presentation: + title_public: Gas price component (PPS) for household consumers - Total price including taxes + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_total_price_including_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_total_price_including_taxes_pps: + title: Gas price component (PPS) for non-household consumers - Total price including taxes + display: + name: Total price including taxes + presentation: + title_public: Gas price component (PPS) for non-household consumers - Total price including taxes + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_total_price_including_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_and_electricity_prices_euro_flat: + title: Gas and electricity prices in Europe + common: + description_short: Prices are given in euros per [megawatt-hour](#dod:watt-hours). They are not adjusted for inflation or differences in living costs between countries. + unit: "current euros per megawatt-hour" + short_unit: "€/MWh" + variables: + electricity_household_all_taxes_and_levies_included_euro: + title: Electricity price (euros) for household consumers - All taxes and levies included + presentation: + title_public: Electricity price for household consumers - All taxes and levies included + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_including_all_taxes}" + electricity_household_excluding_taxes_and_levies_euro: + title: Electricity price (euros) for household consumers - Excluding taxes and levies + presentation: + title_public: Electricity price for household consumers - Excluding taxes and levies + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_taxes}" + electricity_household_excluding_vat_and_other_recoverable_taxes_and_levies_euro: + title: Electricity price (euros) for household consumers - Excluding VAT and other recoverable taxes and levies + presentation: + title_public: Electricity price for household consumers - Excluding VAT and other recoverable taxes and levies + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_vat}" + electricity_non_household_all_taxes_and_levies_included_euro: + title: Electricity price (euros) for non-household consumers - All taxes and levies included + presentation: + title_public: Electricity price for non-household consumers - All taxes and levies included + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_including_all_taxes}" + electricity_non_household_excluding_taxes_and_levies_euro: + title: Electricity price (euros) for non-household consumers - Excluding taxes and levies + presentation: + title_public: Electricity price for non-household consumers - Excluding taxes and levies + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_taxes}" + electricity_non_household_excluding_vat_and_other_recoverable_taxes_and_levies_euro: + title: Electricity price (euros) for non-household consumers - Excluding VAT and other recoverable taxes and levies + presentation: + title_public: Electricity price for non-household consumers - Excluding VAT and other recoverable taxes and levies + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_vat}" + gas_household_all_taxes_and_levies_included_euro: + title: Gas price (euros) for household consumers - All taxes and levies included + presentation: + title_public: Gas price for household consumers - All taxes and levies included + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_including_all_taxes}" + gas_household_excluding_taxes_and_levies_euro: + title: Gas price (euros) for household consumers - Excluding taxes and levies + presentation: + title_public: Gas price for household consumers - Excluding taxes and levies + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_taxes}" + gas_household_excluding_vat_and_other_recoverable_taxes_and_levies_euro: + title: Gas price (euros) for household consumers - Excluding VAT and other recoverable taxes and levies + presentation: + title_public: Gas price for household consumers - Excluding VAT and other recoverable taxes and levies + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_vat}" + gas_non_household_all_taxes_and_levies_included_euro: + title: Gas price (euros) for non-household consumers - All taxes and levies included + presentation: + title_public: Gas price for non-household consumers - All taxes and levies included + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_including_all_taxes}" + gas_non_household_excluding_taxes_and_levies_euro: + title: Gas price (euros) for non-household consumers - Excluding taxes and levies + presentation: + title_public: Gas price for non-household consumers - Excluding taxes and levies + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_taxes}" + gas_non_household_excluding_vat_and_other_recoverable_taxes_and_levies_euro: + title: Gas price (euros) for non-household consumers - Excluding VAT and other recoverable taxes and levies + presentation: + title_public: Gas price for non-household consumers - Excluding VAT and other recoverable taxes and levies + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_vat}" + gas_and_electricity_prices_pps_flat: + title: Gas and electricity prices in Europe (PPS) + common: + description_short: Prices are given in purchasing power standard (PPS) per [megawatt-hour](#dod:watt-hours). + unit: "purchasing power standard per megawatt-hour" + short_unit: "PPS/MWh" + variables: + electricity_household_all_taxes_and_levies_included_pps: + title: Electricity price (PPS) for household consumers - All taxes and levies included + presentation: + title_public: Electricity price for household consumers - All taxes and levies included + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_including_all_taxes}" + electricity_household_excluding_taxes_and_levies_pps: + title: Electricity price (PPS) for household consumers - Excluding taxes and levies + presentation: + title_public: Electricity price for household consumers - Excluding taxes and levies + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_taxes}" + electricity_household_excluding_vat_and_other_recoverable_taxes_and_levies_pps: + title: Electricity price (PPS) for household consumers - Excluding VAT and other recoverable taxes and levies + presentation: + title_public: Electricity price for household consumers - Excluding VAT and other recoverable taxes and levies + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_vat}" + electricity_non_household_all_taxes_and_levies_included_pps: + title: Electricity price (PPS) for non-household consumers - All taxes and levies included + presentation: + title_public: Electricity price for non-household consumers - All taxes and levies included + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_including_all_taxes}" + electricity_non_household_excluding_taxes_and_levies_pps: + title: Electricity price (PPS) for non-household consumers - Excluding taxes and levies + presentation: + title_public: Electricity price for non-household consumers - Excluding taxes and levies + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_taxes}" + electricity_non_household_excluding_vat_and_other_recoverable_taxes_and_levies_pps: + title: Electricity price (PPS) for non-household consumers - Excluding VAT and other recoverable taxes and levies + presentation: + title_public: Electricity price for non-household consumers - Excluding VAT and other recoverable taxes and levies + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_vat}" + gas_household_all_taxes_and_levies_included_pps: + title: Gas price (PPS) for household consumers - All taxes and levies included + presentation: + title_public: Gas price for household consumers - All taxes and levies included + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_including_all_taxes}" + gas_household_excluding_taxes_and_levies_pps: + title: Gas price (PPS) for household consumers - Excluding taxes and levies + presentation: + title_public: Gas price for household consumers - Excluding taxes and levies + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_taxes}" + gas_household_excluding_vat_and_other_recoverable_taxes_and_levies_pps: + title: Gas price (PPS) for household consumers - Excluding VAT and other recoverable taxes and levies + presentation: + title_public: Gas price for household consumers - Excluding VAT and other recoverable taxes and levies + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_vat}" + gas_non_household_all_taxes_and_levies_included_pps: + title: Gas price (PPS) for non-household consumers - All taxes and levies included + presentation: + title_public: Gas price for non-household consumers - All taxes and levies included + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_including_all_taxes}" + gas_non_household_excluding_taxes_and_levies_pps: + title: Gas price (PPS) for non-household consumers - Excluding taxes and levies + presentation: + title_public: Gas price for non-household consumers - Excluding taxes and levies + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_taxes}" + gas_non_household_excluding_vat_and_other_recoverable_taxes_and_levies_pps: + title: Gas price (PPS) for non-household consumers - Excluding VAT and other recoverable taxes and levies + presentation: + title_public: Gas price for non-household consumers - Excluding VAT and other recoverable taxes and levies + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_vat}" diff --git a/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.py b/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.py new file mode 100644 index 00000000000..d6217fcae7a --- /dev/null +++ b/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.py @@ -0,0 +1,896 @@ +"""Load a meadow dataset and create a garden dataset.""" + +from typing import Dict + +import owid.catalog.processing as pr +import pandas as pd +import plotly.express as px +from owid.catalog import Table +from owid.datautils.dataframes import map_series + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +# Dataset codes to select, and their corresponding names. +DATASET_CODES_AND_NAMES = { + #################################################################################################################### + # Gas and electricity prices. + # NOTE: Prices are given per semester. + "nrg_pc_202": "Gas prices for household consumers", # bi-annual data (from 2007) + "nrg_pc_203": "Gas prices for non-household consumers", # bi-annual data (from 2007) + "nrg_pc_204": "Electricity prices for household consumers", # bi-annual data (from 2007) + "nrg_pc_205": "Electricity prices for non-household consumers", # bi-annual data (from 2007) + #################################################################################################################### + # Gas and electricity prices components. + "nrg_pc_202_c": "Gas prices components for household consumers", # annual data (from 2007) + "nrg_pc_203_c": "Gas prices components for non-household consumers", # annual data (from 2007) + "nrg_pc_204_c": "Electricity prices components for household consumers", # annual data (from 2007) + "nrg_pc_205_c": "Electricity prices components for non-household consumers", # annual data (from 2007) + #################################################################################################################### + # Historical data. + # NOTE: For now I think we will have to ignore historical data. + # I doesn't have a band for total price. Instead, it has different consumption bands (defined by "consom"). + # This field is a bit problematic. + # The same value, e.g. "4141050" has different definitions for electricity ("Households - Da (annual consumption: 600 kWh)") and for gas ("Households - D1 (annual consumption: 8.37 GJ)"). + # The fact that the same value is used for different things is inconvenient, but not the main problem. + # The main problem is that we would need to figure out how to properly aggregate these values to get totals (meanwhile current data comes with totals). + # Additionally, historical data is disaggregated in "domestic" and "industrial", whereas current data is split in "households" and "non-households". + # "consom": {} + # "nrg_pc_202_h": "Gas prices for domestic consumers", # bi-annual data (until 2007) + # "nrg_pc_203_h": "Gas prices for industrial consumers", # bi-annual data (until 2007) + # "nrg_pc_204_h": "Electricity prices for domestic consumers", # bi-annual data (until 2007) + # "nrg_pc_205_h": "Electricity prices for industrial consumers", # bi-annual data (until 2007) + # "nrg_pc_206_h": "Electricity marker prices", # bi-annual data (until 2007) + #################################################################################################################### + # Share for transmission and distribution in the network cost for gas and electricity. + # NOTE: Decide if we could use the following. + # "nrg_pc_206": "Share for transmission and distribution in the network cost for gas and electricity", # annual data (from 2007) + #################################################################################################################### + # The following are consumption volumes of electricity by consumption bands. + # It doesn't give the relative consumption of each semester. If I understand correctly, it gives the percentage consumption of each band in the total consumption of the year. + # "nrg_pc_202_v": "Gas consumption volumes for households", # annual data (from 2007) + # "nrg_pc_203_v": "Gas consumption volumes for non-households", # annual data (from 2007) + # "nrg_pc_204_v": "Electricity consumption volumes for households", # annual data (from 2007) + # "nrg_pc_205_v": "Electricity consumption volumes for non-households", # annual data (from 2007) +} + +# Columns to keep and how to rename them. +COLUMNS = { + "nrg_cons": "consumption_band", + "unit": "energy_unit", + "tax": "price_level", + "currency": "currency", + "geo": "country", + "time": "year", + "dataset_code": "dataset_code", + "nrg_prc": "price_component", + "value": "value", +} + +# Mappings of indexes. +# The definitions are copied from (replace [DATASET_CODE] with the dataset code): +# https://ec.europa.eu/eurostat/databrowser/view/[DATASET_CODE]/default/table?lang=en&category=nrg.nrg_price.nrg_pc +INDEXES_MAPPING = { + # Currencies. + "currency": { + "EUR": "Euro", + # Purchasing Power Standard + "PPS": "PPS", + "NAC": "National currency", + "NAT": "National (former) currency", + }, + # Flags (found right next to the value, as a string). + # NOTE: Flag definitions are right below the data table in that page. + "flag": { + "e": "estimated", + "c": "confidential", + "d": "definition differs", + "b": "break in time series", + "p": "provisional", + "u": "low reliability", + "cd": "confidential, definition differs", + # NOTE: I couldn't find the meaning of the following flag. + # It happens for "Electricity prices for non-household consumers" for Cyprus in 2024 (for MWH_GE150000), and all values are zero. + "n": "unknown flag", + }, + # Price levels. + "price_level": { + # All taxes and levies included + "I_TAX": "All taxes and levies included", + # Excluding VAT and other recoverable taxes and levies + # NOTE: This value gives a baseline price for electricity before any additional costs imposed by taxes or fees are added. It represents the net price of electricity. + "X_TAX": "Excluding taxes and levies", + # Excluding value-added tax (VAT) and other recoverable taxes and levies + "X_VAT": "Excluding VAT and other recoverable taxes and levies", + }, + # Consumption bands. + # NOTE: This is only relevant for non-historical data. + "consumption_band": { + # Consumption bands for "Gas prices for household consumers" and "Gas price components for household consumers": + # Consumption of GJ - all bands + "TOT_GJ": "All bands", + # Consumption less than 20 GJ - band D1 + "GJ_LT20": "<20GJ", + # Consumption from 20 GJ to 199 GJ - band D2 + "GJ20-199": "20-199GJ", + # Consumption 200 GJ or over - band D3 + "GJ_GE200": ">=200GJ", + ################################################################################################################ + # Consumption bands for "Gas prices components for non-household consumers" and "Gas prices components for non-household consumers": + # 'TOT_GJ': "All bands", # Already defined above. + # Consumption less than 1 000 GJ - band I1 + "GJ_LT1000": "<1000GJ", + # Consumption from 1 000 GJ to 9 999 GJ -band I2 + "GJ1000-9999": "1000-9999GJ", + # Consumption from 10 000 GJ to 99 999 GJ - band I3 + "GJ10000-99999": "10000-99999GJ", + # Consumption from 100 000 GJ to 999 999 GJ - band I4 + "GJ100000-999999": "100000-999999GJ", + # Consumption from 1 000 000 GJ to 3 999 999 GJ - band I5 + "GJ1000000-3999999": "1000000-3999999GJ", + # Consumption 4 000 000 GJ or over - band I6 + "GJ_GE4000000": ">=4000000GJ", + ################################################################################################################ + # Consumption bands for "Electricity prices for household consumers" and "Electricity prices components for household consumers": + # Consumption of kWh - all bands + "TOT_KWH": "All bands", + # Consumption less than 1 000 kWh - band DA + "KWH_LT1000": "<1000kWh", + # Consumption from 1 000 kWh to 2 499 kWh - band DB + "KWH1000-2499": "1000-2499kWh", + # Consumption from 2 500 kWh to 4 999 kWh - band DC + "KWH2500-4999": "2500-4999kWh", + # Consumption from 5 000 kWh to 14 999 kWh - band DD + "KWH5000-14999": "5000-14999kWh", + # Consumption for 15 000 kWh or over - band DE + "KWH_GE15000": ">=15000kWh", + # NOTE: In the electricity components dataset, there is an additional band, which contains *LE* but in the metadata it seems to correspond to greater or equal, band DE, so it must be a typo in the band name. + # Consumption 15 000 kWh or over - band DE + "KWH_LE15000": ">=15000kWh", + ################################################################################################################ + # Consumption bands for "Electricity prices components for non-household consumers" and "Electricity prices components for non-household consumers": + # Consumption of kWh - all bands + # "TOT_KWH": "All bands", # Already defined above. + # Consumption less than 20 MWh - band IA + "MWH_LT20": "<20MWh", + # Consumption from 20 MWh to 499 MWh - band IB + "MWH20-499": "20-499MWh", + # Consumption from 500 MWh to 1 999 MWh - band IC + "MWH500-1999": "500-1999MWh", + # Consumption from 2 000 MWh to 19 999 MWh - band ID + "MWH2000-19999": "2000-19999MWh", + # Consumption from 20 000 MWh to 69 999 MWh - band IE + "MWH20000-69999": "20000-69999MWh", + # Consumption from 70 000 MWh to 149 999 MWh - band IF + "MWH70000-149999": "70000-149999MWh", + # Consumption 150 000 MWh or over - band IG + "MWH_GE150000": ">=150000MWh", + # NOTE: In the electricity components dataset, there is an additional band: + # Consumption 149 999 MWh or less - bandS IA-IF + "MWH_LE149999": "<=149999MWh", + #################################################################################################################### + }, + # Energy price components. + "price_component": { + # Gas prices components for household and non-household consumers + # Energy and supply + "NRG_SUP": "Energy and supply", + # Network costs + "NETC": "Network costs", + # Taxes, fees, levies and charges + "TAX_FEE_LEV_CHRG": "Taxes, fees, levies, and charges", + # Value added tax (VAT) + "VAT": "Value added tax (VAT)", + # Renewable taxes + "TAX_RNW": "Renewable taxes", + # Capacity taxes + "TAX_CAP": "Capacity taxes", + # Environmental taxes + "TAX_ENV": "Environmental taxes", + # Renewable taxes allowance + "TAX_RNW_ALLOW": "Renewable taxes allowance", + # Capacity taxes allowances + "TAX_CAP_ALLOW": "Capacity taxes allowances", + # Environmental taxes allowance + "TAX_ENV_ALLOW": "Environmental taxes allowance", + # Other allowance + "ALLOW_OTH": "Other allowance", + # Other + "OTH": "Other", + # Electricity prices components for household and non-household consumers + # All the above, plus the additional: + # Nuclear taxes + "TAX_NUC": "Nuclear taxes", + # Nuclear taxes allowance + "TAX_NUC_ALLOW": "Nuclear taxes allowance", + # Taxes, fees, levies and charges allowance + "TAX_FEE_LEV_CHRG_ALLOW": "Taxes, fees, levies, and charges allowance", + # From the metadata page (https://ec.europa.eu/eurostat/cache/metadata/en/nrg_pc_204_sims.htm), these are the components: + # * Energy and supply: generation, aggregation, balancing energy, supplied energy costs, customer services, after-sales management and other supply costs. + # * Network cost: transmission and distribution tariffs, transmission and distribution losses, network costs, after-sale service costs, system service costs, and meter rental and metering costs. + # * Value added taxes (VAT): as defined in Council Directive 2006/112/EC. + # * Renewable taxes: taxes, fees, levies or charges relating to the promotion of renewable energy sources, energy efficiency and CHP generation. + # * Capacity taxes: Taxes, fees, levies or charges relating to capacity payments, energy security and generation adequacy; taxes on coal industry restructuring; taxes on electricity distribution; stranded costs and levies on financing energy regulatory authorities or market and system operators. + # * Environmental taxes: taxes, fees, levies or charges relating to air quality and for other environmental purposes; taxes on emissions of CO2 or other greenhouse gases. This component includes the excise duties. + # * Nuclear taxes: taxes, fees, levies or charges relating to the nuclear sector, including nuclear decommissioning, inspections and fees for nuclear installations. + # * All other taxes: taxes, fees, levies or charges not covered by any of the previous five categories: support for district heating; local or regional fiscal charges; island compensation; concession fees relating to licences and fees for the occupation of land and public or private property by networks or other devices. + }, + # Energy units. + "energy_unit": { + # Gigajoule (gross calorific value - GCV) + "GJ_GCV": "GJ", + # Kilowatt-hour + "KWH": "kWh", + # The following is used in consumption volumes datasets. + # "PC": "Percentage", + }, +} + +# Dataset codes for prices and components. +DATASET_CODES_PRICES = ["nrg_pc_202", "nrg_pc_203", "nrg_pc_204", "nrg_pc_205"] +DATASET_CODES_COMPONENTS = ["nrg_pc_202_c", "nrg_pc_203_c", "nrg_pc_204_c", "nrg_pc_205_c"] +DATASET_CODE_TO_ENERGY_SOURCE = { + "nrg_pc_202": "Gas", + "nrg_pc_203": "Gas", + "nrg_pc_204": "Electricity", + "nrg_pc_205": "Electricity", + "nrg_pc_202_c": "Gas", + "nrg_pc_203_c": "Gas", + "nrg_pc_204_c": "Electricity", + "nrg_pc_205_c": "Electricity", +} +DATASET_CODE_TO_CONSUMER_TYPE_MAPPING = { + "nrg_pc_202": "Household", + "nrg_pc_203": "Non-household", + "nrg_pc_204": "Household", + "nrg_pc_205": "Non-household", + "nrg_pc_202_c": "Household", + "nrg_pc_203_c": "Non-household", + "nrg_pc_204_c": "Household", + "nrg_pc_205_c": "Non-household", +} + + +# The following components need to be present in the prices components datasets of a country-year-dataset-currency, otherwise its data will not be included. +MANDATORY_PRICE_COMPONENTS = [ + "Energy and supply", + "Network costs", + "Taxes, fees, levies, and charges", +] + +# List of components that add up to the total price. +# NOTE: See find_best_combination_of_components to understand how this choice was made. +COMPONENTS_THAT_ADD_UP_TO_TOTAL = ["Energy and supply", "Network costs", "Taxes, fees, levies, and charges"] + +# Label to use for the calculated total price based on the sum of the main components. +COMPONENTS_TOTAL_PRICE_LABEL = "Total price, including taxes" + + +def sanity_check_inputs(tb: Table) -> None: + # Ensure all relevant dataset codes are present. + error = "Some dataset codes are missing." + assert set(DATASET_CODES_AND_NAMES) <= set(tb["dataset_code"]), error + # Check that each dataset has only one value in fields "freq", "product", and "nrg_cons". + # error = "Some datasets have more than one value in field 'freq'." + # assert (tb.groupby("dataset_code")["freq"].nunique() == 1).all(), error + # error = "Expected 'freq' column to be either A (annual) or S (bi-annual)." + # assert set(tb["freq"].dropna()) == set(["A", "S"]), error + # error = "Some datasets have more than one value in field 'product'." + # assert (tb.dropna(subset="product").groupby("dataset_code")["product"].nunique() == 1).all(), error + # error = "Expected 'product' column to be either 4100 (gas) or 6000 (electricity)." + # assert set(tb["product"].dropna()) == set([4100, 6000]), error + error = "Expected electricity prices to be measured in kWh." + assert set( + tb[tb["dataset_code"].isin(["nrg_pc_204", "nrg_pc_205", "nrg_pc_204_h", "nrg_pc_205_h", "nrg_pc_206_h"])][ + "energy_unit" + ] + ) == set(["KWH"]), error + # error = "Expected 'customer' column to be empty, for the selected datasets." + # assert set(tb["customer"].dropna()) == set(), error + # error = "Expected 'consom' column to be empty, for the selected datasets (that column is only relevant for historical data)." + # assert set(tb["consom"].dropna()) == set(), error + for field, mapping in INDEXES_MAPPING.items(): + if field == "flag": + # Flags need to first be extracted from the value (so they will be sanity checked later). + continue + error = f"Unexpected values in field '{field}'." + assert set(tb[field].dropna()) == set(mapping), error + + +def prepare_inputs(tb: Table) -> Table: + # Values sometimes include a letter, which is a flag. Extract those letters and create a separate column with them. + # Note that sometimes there can be multiple letters (which means multiple flags). + tb["flag"] = tb["value"].astype("string").str.extract(r"([a-z]+)", expand=False) + tb["value"] = tb["value"].str.replace(r"[a-z]", "", regex=True) + + # Some values are start with ':' (namely ':', ': ', ': c', ': u', ': cd'). Replace them with nan. + tb.loc[tb["value"].str.startswith(":"), "value"] = None + + # Assign a proper type to the column of values. + tb["value"] = tb["value"].astype(float) + + # Create a clean column of years, and another of dates. + tb["year-semester"] = tb["year"].str.strip().copy() + tb["year"] = tb["year-semester"].str[0:4].astype(int) + # For the date column: + # * For the first semester, use April 1st. + # * For the second semester, use October 1st. + # * For annual data, use July 1st. + semester_1_mask = tb["year-semester"].str.contains("S1") + semester_2_mask = tb["year-semester"].str.contains("S2") + annual_mask = tb["year-semester"].str.isdigit() + error = "Unexpected values in field 'year-semester'." + assert (semester_1_mask | semester_2_mask | annual_mask).all(), error + tb["date"] = pd.to_datetime(tb["year"].astype(str) + "-07-01") + tb.loc[semester_1_mask, "date"] = pd.to_datetime(tb[semester_1_mask]["year"].astype(str) + "-04-01") + tb.loc[semester_2_mask, "date"] = pd.to_datetime(tb[semester_2_mask]["year"].astype(str) + "-10-01") + + return tb + + +def harmonize_indexes_and_countries(tb: Table) -> Table: + # Add a column with the dataset name. + tb["dataset_name"] = map_series( + tb["dataset_code"], + mapping=DATASET_CODES_AND_NAMES, + warn_on_missing_mappings=True, + warn_on_unused_mappings=True, + show_full_warning=True, + ) + + # Harmonize all other index names. + for field, mapping in INDEXES_MAPPING.items(): + # Avoid categorical dtypes. + tb[field] = tb[field].astype("string") + not_null_mask = tb[field].notnull() + tb.loc[not_null_mask, field] = map_series( + tb[not_null_mask][field], + mapping=mapping, + warn_on_missing_mappings=True, + warn_on_unused_mappings=True, + show_full_warning=True, + ) + + # Harmonize country names. + # Countries are given in NUTS (Nomenclature of Territorial Units for Statistics) codes. + # Region codes are defined in: https://ec.europa.eu/eurostat/web/nuts/correspondence-tables + # There are additional codes not included there, namely: + # EA: Countries in the Euro Area, that use the Euro as their official currency. + # In the historical datasets, there are some additional regions: + # EU15: The 15 countries that made up the EU prior to its 2004 expansion. + # EU25: The 25 member states after the 2004 enlargement, which added ten countries. + # EU27_2007: The 27 EU member states in 2007. + # EU27_2020: The 27 EU members after the United Kingdom left in 2020. + # UA: Ukraine (not a member of the EU, but often included in some European data). + # UK: United Kingdom (not a member since 2020, but included in some European data). + tb = geo.harmonize_countries( + df=tb, countries_file=paths.country_mapping_path, excluded_countries_file=paths.excluded_countries_path + ) + + return tb + + +######################################################################################################################## + +# The code in this block is not used in the data pipeline, but it was useful to understand the data and justify some of the choices. + + +def compare_components_and_prices_data(tb: Table) -> None: + # Compare biannual prices data (without averaging overs semesters) with annual components data. + price_level = "All taxes and levies included" + tb_biannual = tb[(tb["year-semester"].str.contains("S")) & (tb["currency"] == "Euro")].reset_index(drop=True) + tb_biannual = tb_biannual[(tb_biannual["price_component_or_level"] == price_level)][ + ["dataset_code", "country", "date", "value"] + ] + # Similarly, for annual data, assign July 1st. + tb_annual = tb[(~tb["year-semester"].str.contains("S")) & (tb["currency"] == "Euro")].reset_index(drop=True) + tb_annual["dataset_code"] = tb_annual["dataset_code"].str.replace("_c", "") + + combination = COMPONENTS_THAT_ADD_UP_TO_TOTAL + annual_components = ( + tb_annual[(tb_annual["price_component_or_level"].isin(combination))] + .groupby( + ["dataset_code", "country", "date"], + observed=True, + as_index=False, + ) + .agg({"value": lambda x: x.sum(min_count=1)}) + .dropna() + .reset_index(drop=True) + ) + + # Combine both datasets for plotting. + compared = pd.concat( + [annual_components.assign(**{"source": "components"}), tb_biannual.assign(**{"source": "prices"})], + ignore_index=True, + ) + # Only a few country-years could be compared this way. Most of the points in the prices datasets were missing. + for dataset_code in compared["dataset_code"].unique(): + for country in sorted(set(compared["country"])): + _compared = compared[(compared["dataset_code"] == dataset_code) & (compared["country"] == country)] + if len(set(_compared["source"])) < 2: + continue + px.line( + _compared, + x="date", + y="value", + color="source", + markers=True, + title=f"{dataset_code} - {country}", + ).update_yaxes(range=[0, None]).show() + + +def find_best_combination_of_components(tb: Table) -> None: + # Check that the resulting total price for the components dataset (summing up components) is similar to the biannual electricity prices data. + + # Ideally, the prices obtained by adding up components (in the components dataset) should be similar to those obtained in the prices dataset. + # However, both are very sparse (especially the prices dataset), and the prices dataset is also given in semesters, which makes it difficult to compare (without having the actual consumption of each semester to be able to compute a weighted average). + # Transforming biannual data into annual data is not straightforward. + # I tried simply taking the average, but what I found is that the annual components prices (summed over all components) tends to be systematically higher than the biannual prices (averaged over the two semester of the year). I suppose this was caused by doing a simple average instead of weighting by consumption. In semesters with higher consumption (e.g. winter), the increased demand tends to drive prices up. Annual prices, as far as I understand, are consumption-weighted averages, and therefore assign a larger weight to those semesters with higher prices. So, intuitively, it makes sense that the true annual prices tend to be higher than the averaged biannual prices. + # We could create a weighted average, but we would need the actual consumption of each semester (which I haven't found straightaway). + + # Compute an annual average only if there is data for the two semesters. + price_level = "All taxes and levies included" + tb_biannual = tb[(tb["year-semester"].str.contains("S")) & (tb["currency"] == "Euro")].reset_index(drop=True) + tb_biannual_filtered = ( + tb_biannual.dropna(subset="value") + .groupby( + ["country", "year", "dataset_code", "price_component_or_level"], + observed=True, + as_index=False, + ) + .filter(lambda x: len(x) == 2) + ) + tb_biannual = tb_biannual_filtered.groupby( + ["country", "year", "dataset_code", "price_component_or_level"], + observed=True, + as_index=False, + ).agg({"value": "mean"}) + tb_biannual = tb_biannual[(tb_biannual["price_component_or_level"] == price_level)][ + ["dataset_code", "country", "year", "value"] + ] + # Similarly, for annual data, assign July 1st. + tb_annual = tb[(~tb["year-semester"].str.contains("S")) & (tb["currency"] == "Euro")].reset_index(drop=True) + tb_annual["dataset_code"] = tb_annual["dataset_code"].str.replace("_c", "") + + def _get_annual_sum(tb_annual, combination): + annual_components = ( + tb_annual[(tb_annual["price_component_or_level"].isin(combination))] + .groupby( + ["dataset_code", "country", "year"], + observed=True, + as_index=False, + ) + .agg({"value": lambda x: x.sum(min_count=1)}) + .dropna() + .reset_index(drop=True) + ) + + return annual_components + + import itertools + + from tqdm.auto import tqdm + + # Get all possible combinations of components. + elements = INDEXES_MAPPING["price_component"].values() + combinations = [] + for r in range(1, len(elements) + 1): + combinations.extend(itertools.combinations(elements, r)) + # Keep only combinations that include "Energy and supply" and "Network costs". + combinations = [c for c in combinations if "Energy and supply" in c and "Network costs" in c] + + # Brute-force analysis: Check which combination of components minimizes the error between the sum of components and the prices data. + # NOTE: This takes about 4 minutes. + error_median = [] + error_mean = [] + error_max = [] + for combination in tqdm(combinations): + annual_components = _get_annual_sum(tb_annual, combination) + compared = ( + tb_biannual.merge( + annual_components, + on=["dataset_code", "country", "year"], + how="inner", + suffixes=("_prices", "_components"), + ) + .dropna() + .reset_index(drop=True) + ) + compared["pct"] = 100 * abs(compared["value_prices"] - compared["value_components"]) / compared["value_prices"] + error_mean.append(compared["pct"].mean()) + error_median.append(compared["pct"].median()) + error_max.append(compared["pct"].max()) + # Find the combination that minimizes the error. + results_df = pd.DataFrame( + { + "combination": combinations, + "error_mean": error_mean, + "error_median": error_median, + "error_max": error_max, + } + ) + # There is no single combination that minimizes all error. + set(results_df[results_df["error_mean"] == results_df["error_mean"].min()]["combination"]) + # After inspection, there are different combinations that minimize error (since some components are actually always zero). In terms of the minimum mean error, the combinations are: + # ('Energy and supply', 'Network costs', 'Taxes, fees, levies, and charges'), + # ('Energy and supply', 'Network costs', 'Taxes, fees, levies, and charges', 'Capacity taxes allowances'), + # ('Energy and supply', 'Network costs', 'Taxes, fees, levies, and charges', 'Capacity taxes allowances', 'Nuclear taxes allowance'), + # ('Energy and supply', 'Network costs', 'Taxes, fees, levies, and charges', 'Nuclear taxes allowance') + # Given that some of those allowance components are actually (almost) always zero, it seems clear that + # the best combination is, as expected: + components_optimal = ["Energy and supply", "Network costs", "Taxes, fees, levies, and charges"] + annual_components = _get_annual_sum(tb_annual, combination=components_optimal) + compared = ( + tb_biannual.merge( + annual_components, on=["dataset_code", "country", "year"], how="inner", suffixes=("_prices", "_components") + ) + .dropna() + .reset_index(drop=True) + ) + compared["pct"] = 100 * abs(compared["value_prices"] - compared["value_components"]) / compared["value_prices"] + compared.sort_values("pct", ascending=False).head(60) + # For most countries, the agreement is good, but some country-years, the discrepancy is significant, e.g. Denmark non-household electricity in 2022 and 2023, with an error of 26%. + # There are only a few other discrepancies above 10%. + + # Visually inspect these discrepancies. + compared = pd.concat( + [annual_components.assign(**{"source": "components"}), tb_biannual.assign(**{"source": "prices"})], + ignore_index=True, + ) + # Only a few country-years could be compared this way. Most of the points in the prices datasets were missing. + for dataset_code in compared["dataset_code"].unique(): + for country in sorted(set(compared["country"])): + if ( + len( + set( + compared[(compared["dataset_code"] == dataset_code) & (compared["country"] == country)][ + "source" + ] + ) + ) + < 2 + ): + continue + px.line( + compared[(compared["dataset_code"] == dataset_code) & (compared["country"] == country)], + x="year", + y="value", + color="source", + markers=True, + title=f"{dataset_code} - {country}", + ).update_yaxes(range=[0, None]).show() + + # Conclusions: + # * The prices and components datasets coincide reasonably well. To recover prices, it seems that the components to be added up are just "Energy and supply", "Network costs", and "Taxes, fees, levies, and charges". For most countries, this combination gives a good agreement with the prices dataset. However, for some countries, there is a significant discrepancy. + # * Numerically, I have checked that for all price components datasets, "Taxes, fees, levies and charges" coincides with the sum of 'Capacity taxes', 'Environmental taxes', 'Nuclear taxes', 'Renewable taxes', 'Value added tax (VAT)', 'Other'. For some country-years, there is a small discrepancy. + # * What's not so clear is what happens with the "allowances". Is "Taxes, fees, levies, and charges allowance" the sum of all other "* allowance"? It's hard to know, since it's non-zero only once (nrg_pc_204_c Netherlands 2023). At that point, it does coincide with the sum of all other "* allowance". But there are other instances of non-zero "* allowance" where "Taxes...allowance" is not defined. It may be possible that allowances are not included in the prices dataset. + + +def plot_final_comparison_between_prices_and_components_data(tb: Table) -> None: + for country in tb["country"].unique(): + for consumer_type in tb["consumer_type"].unique(): + for source in tb["source"].unique(): + _tb = tb[ + (tb["country"] == country) + & (tb["source"] == source) + & (tb["consumer_type"] == consumer_type) + & ( + tb["price_component_or_level"].isin( + [COMPONENTS_TOTAL_PRICE_LABEL, "All taxes and levies included"] + ) + ) + ] + if len(_tb["price_component_or_level"].unique()) < 2: + continue + px.line( + _tb, + x="date", + y="price_euro", + color="price_component_or_level", + markers=True, + title=f"{consumer_type} {source} - {country}", + ).show() + + +######################################################################################################################## + + +def select_and_prepare_relevant_data(tb: Table) -> Table: + # All datasets have a energy unit except electricity components (both for household and non-households). + # I assume the energy unit is kWh. + error = "Expected electricity components (both for household and non-households) to have no energy unit. Remove this code." + assert tb[tb["dataset_code"].isin(["nrg_pc_204_c", "nrg_pc_205_c"])]["energy_unit"].isnull().all(), error + tb.loc[tb["dataset_code"].isin(["nrg_pc_204_c", "nrg_pc_205_c"]), "energy_unit"] = "kWh" + + error = "Expected all datasets to have the same energy unit (kWh)." + assert ( + tb.groupby(["dataset_code"], observed=True, as_index=False) + .agg({"energy_unit": lambda x: "kWh" in x.unique()})["energy_unit"] + .all() + ), error + # Select the same energy unit for all datasets (kWh). + tb = tb[tb["energy_unit"] == "kWh"].drop(columns=["energy_unit"], errors="raise").reset_index(drop=True) + + # Convert prices from price per kWh to price per MWh. + tb["value"] *= 1000 + + # For convenience, instead of having a column for price component (for components datasets) and price level (for prices datasets), create a single column with the price component or level. + assert tb[(tb["price_level"].isnull()) & (tb["price_component"].isnull())].empty + assert tb[(tb["price_level"].notnull()) & (tb["price_component"].notnull())].empty + tb["price_component_or_level"] = tb["price_level"].fillna(tb["price_component"]) + tb = tb.drop(columns=["price_level", "price_component"], errors="raise") + + # After inspection, it looks like the "All bands" consumption is very sparse in the prices datasets. + # One option (if we decided to use the prices dataset) would be to use the more common consumption bands only, which are better informed. + # In the components dataset, "All bands" seems to be less sparse (at least from 2019 onwards). + # To get the total price from the components dataset, we would need to add up components. + # But we would need to figure out which one is the subset of components that ensures no double-counting. + tb = ( + tb[tb["consumption_band"] == "All bands"] + .drop(columns=["consumption_band"], errors="raise") + .reset_index(drop=True) + ) + + # Find the combination of price components that needs to be summed up to recover the full prices. + # NOTE: Uncomment to perform the analysis again, and see conclusions in the following function to understand the choices. + # find_best_combination_of_components(tb=tb) + + # Visually compare the resulting prices obtained by adding up certain components, with the original prices data. + # NOTE: Uncomment to perform some visual checks. + # compare_components_and_prices_data(tb=tb) + + # Remove groups (of country-year-dataset-currency) from the components dataset for which certain components (e.g. "Energy and supply") are not included. + # For example, Albania doesn't have "Energy and supply" costs for household electricity, but it does have other components (e.g. "Network costs"). + tb.loc[ + (tb["dataset_code"].isin(DATASET_CODES_COMPONENTS)) + & ( + ~tb.groupby(["country", "year", "currency"])["price_component_or_level"].transform( + lambda x: all(comp in x.tolist() for comp in MANDATORY_PRICE_COMPONENTS) + ) + ), + "value", + ] = None + + # Remove empty rows. + tb = tb.dropna(subset=["value"]).reset_index(drop=True) + + # Remove data with certain flags. + tb = tb[ + ~tb["flag"].isin( + [ + "confidential", + "definition differs", + "low reliability", + "confidential, definition differs", + "unknown flag", + ] + ) + ].reset_index(drop=True) + error = "Unexpected flag values." + assert set(tb["flag"].dropna()) <= set(["estimated", "break in time series", "provisional", "unknown flag"]), error + + # Add total price to the components dataset, by adding up the contribution of the main components. + tb_components_total = ( + tb[ + tb["dataset_code"].isin(DATASET_CODES_COMPONENTS) + & (tb["price_component_or_level"].isin(COMPONENTS_THAT_ADD_UP_TO_TOTAL)) + ] + .groupby( + ["currency", "country", "year", "dataset_code", "year-semester", "date", "dataset_name"], + observed=True, + as_index=False, + ) + .agg({"value": "sum"}) + .assign(**{"price_component_or_level": COMPONENTS_TOTAL_PRICE_LABEL}) + ) + tb = pr.concat([tb, tb_components_total], ignore_index=True) + + # Create a column for the energy source. + tb["source"] = map_series( + tb["dataset_code"], + mapping=DATASET_CODE_TO_ENERGY_SOURCE, + warn_on_missing_mappings=True, + warn_on_unused_mappings=True, + show_full_warning=True, + ) + error = "Unexpected energy source." + assert set(tb["source"]) == set(["Gas", "Electricity"]), error + + # Create a column for the consumer type. + tb["consumer_type"] = map_series( + tb["dataset_code"], + mapping=DATASET_CODE_TO_CONSUMER_TYPE_MAPPING, + warn_on_missing_mappings=True, + warn_on_unused_mappings=True, + show_full_warning=True, + ) + error = "Unexpected consumer type." + assert set(tb["consumer_type"]) == set(["Household", "Non-household"]), error + + # Drop unnecessary columns. + tb = tb.drop(columns=["flag", "year-semester", "dataset_name"], errors="raise") + + # It would be confusing to keep different national currencies, so, keep only Euro and PPS. + tb = tb[tb["currency"].isin(["Euro", "PPS"])].reset_index(drop=True) + + # Separate euros and PPS in two different columns. + tb = ( + tb[tb["currency"] == "Euro"] + .drop(columns=["currency"]) + .merge( + tb[tb["currency"] == "PPS"].drop(columns=["currency"]), + how="outer", + on=["country", "year", "date", "dataset_code", "source", "price_component_or_level", "consumer_type"], + suffixes=("_euro", "_pps"), + ) + .rename(columns={"value_euro": "price_euro", "value_pps": "price_pps"}, errors="raise") + ) + + return tb + + +def prepare_wide_tables(tb: Table) -> Dict[str, Table]: + # Table for average prices (in euros) of gas and electricity prices of household and non-household consumers. + tb_prices_euro = tb[tb["dataset_code"].isin(DATASET_CODES_PRICES)].pivot( + index=["country", "date"], + columns=["source", "consumer_type", "price_component_or_level"], + values="price_euro", + join_column_levels_with="-", + ) + # Table for average prices (in PPS) of gas and electricity prices of household and non-household consumers. + tb_prices_pps = tb[tb["dataset_code"].isin(DATASET_CODES_PRICES)].pivot( + index=["country", "date"], + columns=["source", "consumer_type", "price_component_or_level"], + values="price_pps", + join_column_levels_with="-", + ) + # Improve tables format. + tb_prices_euro = tb_prices_euro.format(["country", "date"], short_name="gas_and_electricity_prices_euro_flat") + tb_prices_pps = tb_prices_pps.format(["country", "date"], short_name="gas_and_electricity_prices_pps_flat") + + # Improve column names. + tb_prices_euro = tb_prices_euro.rename( + columns={column: column.replace("__", "_") + "_euro" for column in tb_prices_euro.columns}, errors="raise" + ) + tb_prices_pps = tb_prices_pps.rename( + columns={column: column.replace("__", "_") + "_pps" for column in tb_prices_pps.columns}, errors="raise" + ) + + # Table for price components (in euros) of gas and electricity prices of household and non-household consumers. + tb_price_components_euro = tb[tb["dataset_code"].isin(DATASET_CODES_COMPONENTS)].pivot( + index=["country", "year"], + columns=["source", "consumer_type", "price_component_or_level"], + values="price_euro", + join_column_levels_with="-", + ) + # Table for price components (in PPS) of gas and electricity prices of household and non-household consumers. + tb_price_components_pps = tb[tb["dataset_code"].isin(DATASET_CODES_COMPONENTS)].pivot( + index=["country", "year"], + columns=["source", "consumer_type", "price_component_or_level"], + values="price_pps", + join_column_levels_with="-", + ) + # Improve tables format. + tb_price_components_euro = tb_price_components_euro.format( + ["country", "year"], short_name="gas_and_electricity_price_components_euro_flat" + ) + tb_price_components_pps = tb_price_components_pps.format( + ["country", "year"], short_name="gas_and_electricity_price_components_pps_flat" + ) + + # Improve column names. + tb_price_components_euro = tb_price_components_euro.rename( + columns={column: column.replace("__", "_") + "_euro" for column in tb_price_components_euro.columns}, + errors="raise", + ) + tb_price_components_pps = tb_price_components_pps.rename( + columns={column: column.replace("__", "_") + "_pps" for column in tb_price_components_pps.columns}, + errors="raise", + ) + + return tb_prices_euro, tb_prices_pps, tb_price_components_euro, tb_price_components_pps + + +def sanity_check_outputs(tb: Table) -> None: + error = "Expected 'Energy and supply' and 'Network costs' to be non-negative." + assert tb[ + tb["dataset_code"].isin(DATASET_CODES_COMPONENTS) + & tb["price_component_or_level"].isin(["Energy and supply", "Network costs"]) + & (tb["price_euro"] < 0) + ].empty, error + + # Further sanity checks on component prices. + tb_components = tb[tb["dataset_code"].isin(DATASET_CODES_COMPONENTS)].reset_index(drop=True) + tb_taxes_sum = ( + tb_components[ + tb_components["price_component_or_level"].isin( + [ + "Capacity taxes", + "Environmental taxes", + "Nuclear taxes", + "Renewable taxes", + "Value added tax (VAT)", + "Other", + ] + ) + ] + .groupby(["dataset_code", "country", "year"], observed=True, as_index=False) + .agg({"price_euro": "sum"}) + ) + tb_taxes_original = tb_components[ + tb_components["price_component_or_level"] == "Taxes, fees, levies, and charges" + ].reset_index(drop=True)[["dataset_code", "country", "year", "price_euro"]] + # NOTE: The median value of the sum is 0.0191 euros/kWh. When comparing the percentage difference, ignore values that are too small. + compared = tb_taxes_sum.merge( + tb_taxes_original, how="outer", on=["dataset_code", "country", "year"], suffixes=("_sum", "_original") + ) + compared["dev"] = 100 * ( + abs(compared["price_euro_sum"] - compared["price_euro_original"]) / compared["price_euro_original"] + ) + error = "Expected the sum of 'Capacity taxes', 'Environmental taxes', 'Nuclear taxes', 'Renewable taxes', 'Value added tax (VAT)', 'Other' to coincide with 'Taxes, fees, levies, and charges', within 2% (ignoring any prices below 0.007, which is 17% of rows)." + # NOTE: Some dataset-country-year have a significant discrepancy, e.g. nrg_pc_202_c-Greece-2022, with a price of 6.7€/MWh. + assert compared[(compared["price_euro_original"] > 7) & (compared["dev"] > 2)].empty, error + # compared.sort_values("dev", ascending=False).head(60) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("gas_and_electricity_prices") + + # Read table from meadow dataset. + tb = ds_meadow.read("gas_and_electricity_prices") + + # + # Process data. + # + # Select relevant dataset codes, and add a column with the dataset name. + tb = tb[tb["dataset_code"].isin(DATASET_CODES_AND_NAMES.keys())].reset_index(drop=True) + + # Select and rename columns. + tb = tb[list(COLUMNS)].rename(columns=COLUMNS, errors="raise") + + # Sanity checks on inputs. + sanity_check_inputs(tb=tb) + + # Clean inputs. + tb = prepare_inputs(tb=tb) + + # Harmonize indexes and country names. + tb = harmonize_indexes_and_countries(tb=tb) + + # Select and prepare relevant data. + tb = select_and_prepare_relevant_data(tb=tb) + + # Sanity check outputs. + sanity_check_outputs(tb=tb) + + # Uncomment to plot a comparison (for each country, source, and consumer type) between the prices and the components data. + # NOTE: Some of the biggest discrepancies happen where prices data is given only for one of the semesters. This is the case of Georgia household electricity in 2021 and 2022, where we can't see the value of the missing semester (which could explain why the components data is significantly higher). + # plot_final_comparison_between_prices_and_components_data(tb=tb) + + # Create convenient wide tables. + tb_prices_euro, tb_prices_pps, tb_price_components_euro, tb_price_components_pps = prepare_wide_tables(tb=tb) + + # Improve main table format. + tb = tb.drop(columns=["dataset_code"]).format( + ["country", "date", "source", "consumer_type", "price_component_or_level"] + ) + + # + # Save outputs. + # + # Create a new garden dataset. + ds_garden = create_dataset( + dest_dir, + tables=[tb, tb_prices_euro, tb_prices_pps, tb_price_components_euro, tb_price_components_pps], + check_variables_metadata=True, + default_metadata=ds_meadow.metadata, + ) + ds_garden.save() diff --git a/etl/steps/data/garden/excess_mortality/latest/excess_mortality/__init__.py b/etl/steps/data/garden/excess_mortality/latest/excess_mortality/__init__.py index 9811af8f7d0..c8c77fe03b7 100644 --- a/etl/steps/data/garden/excess_mortality/latest/excess_mortality/__init__.py +++ b/etl/steps/data/garden/excess_mortality/latest/excess_mortality/__init__.py @@ -6,6 +6,7 @@ This step merges the two datasets into one single dataset, combining metrics from both sources to obtain excess mortality metrics. """ + from input import build_df from owid.catalog import Table from process import process_df diff --git a/etl/steps/data/garden/excess_mortality/latest/excess_mortality/process.py b/etl/steps/data/garden/excess_mortality/latest/excess_mortality/process.py index be8f5eb6d23..390bb67ab8d 100644 --- a/etl/steps/data/garden/excess_mortality/latest/excess_mortality/process.py +++ b/etl/steps/data/garden/excess_mortality/latest/excess_mortality/process.py @@ -1,4 +1,5 @@ """Processing tools.""" + from datetime import datetime, timedelta import numpy as np diff --git a/etl/steps/data/garden/excess_mortality/latest/hmd_stmf.py b/etl/steps/data/garden/excess_mortality/latest/hmd_stmf.py index 7e6e50dead8..1630b8fa7b8 100644 --- a/etl/steps/data/garden/excess_mortality/latest/hmd_stmf.py +++ b/etl/steps/data/garden/excess_mortality/latest/hmd_stmf.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + from datetime import date import pandas as pd diff --git a/etl/steps/data/garden/excess_mortality/latest/xm_karlinsky_kobak.py b/etl/steps/data/garden/excess_mortality/latest/xm_karlinsky_kobak.py index 4a940cfef89..0afbf1a0f01 100644 --- a/etl/steps/data/garden/excess_mortality/latest/xm_karlinsky_kobak.py +++ b/etl/steps/data/garden/excess_mortality/latest/xm_karlinsky_kobak.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + from typing import Any, Dict, List import pandas as pd diff --git a/etl/steps/data/garden/faostat/2023-02-22/detected_anomalies.py b/etl/steps/data/garden/faostat/2023-02-22/detected_anomalies.py index 22d10b928cd..f484e961f1b 100644 --- a/etl/steps/data/garden/faostat/2023-02-22/detected_anomalies.py +++ b/etl/steps/data/garden/faostat/2023-02-22/detected_anomalies.py @@ -5,6 +5,7 @@ See documentation of class DataAnomaly below for more details on how anomaly classes are structured. """ + import abc import os from typing import Tuple diff --git a/etl/steps/data/garden/faostat/2023-06-12/additional_variables.py b/etl/steps/data/garden/faostat/2023-06-12/additional_variables.py index 9c45c5a7913..4011fad9515 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/additional_variables.py +++ b/etl/steps/data/garden/faostat/2023-06-12/additional_variables.py @@ -1,6 +1,4 @@ -"""Dataset that combines different variables of other FAOSTAT datasets. - -""" +"""Dataset that combines different variables of other FAOSTAT datasets.""" import numpy as np import pandas as pd diff --git a/etl/steps/data/garden/faostat/2023-06-12/detected_anomalies.py b/etl/steps/data/garden/faostat/2023-06-12/detected_anomalies.py index 22d10b928cd..f484e961f1b 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/detected_anomalies.py +++ b/etl/steps/data/garden/faostat/2023-06-12/detected_anomalies.py @@ -5,6 +5,7 @@ See documentation of class DataAnomaly below for more details on how anomaly classes are structured. """ + import abc import os from typing import Tuple diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_cahd.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_cahd.py index 048afae3560..02734da3633 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_cahd.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_cahd.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_cahd dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_ef.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_ef.py index 9969bfd421f..183b0f6013f 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_ef.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_ef.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_ef dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_ei.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_ei.py index 5bd7d23db88..1a013f6e824 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_ei.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_ei.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_ei dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_ek.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_ek.py index c6ec4c862e8..bb2753d1d0a 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_ek.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_ek.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_ek dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_el.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_el.py index 43b06ade38c..87263c413ed 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_el.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_el.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_el dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_emn.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_emn.py index 5f12637ec70..e2b1eaff5d3 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_emn.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_emn.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_emn dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_ep.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_ep.py index 0a44564deea..0053f94814b 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_ep.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_ep.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_ep dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_esb.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_esb.py index 2a4896b9edf..10fb21b0187 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_esb.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_esb.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_esb dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_fa.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_fa.py index e594773a567..c0c7f77fbfb 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_fa.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_fa.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_fa dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_fo.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_fo.py index bba98a5e224..7d4fac7b2d7 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_fo.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_fo.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_fo dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_fs.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_fs.py index a836381fb94..3ea0dd31e98 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_fs.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_fs.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_fs dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_gn.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_gn.py index 4d77b41c597..93bacc020e3 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_gn.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_gn.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_gn dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_ic.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_ic.py index 5e86234ddde..69e2f85c84c 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_ic.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_ic.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_ic dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_lc.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_lc.py index ab508fd95ad..76ca635f1f8 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_lc.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_lc.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_lc dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_qi.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_qi.py index 8c271f07bc2..f671c690199 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_qi.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_qi.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_qi dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_qv.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_qv.py index f564688376e..cb89edeef24 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_qv.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_qv.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_qv dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_rfb.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_rfb.py index 68669b4cbd4..d3300e2ef69 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_rfb.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_rfb.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_rfb dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_rfn.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_rfn.py index 4ebfe341728..20bc8770f77 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_rfn.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_rfn.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_rfn dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_rl.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_rl.py index f43cbe31912..0eea09d6235 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_rl.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_rl.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_rl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_rp.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_rp.py index f15e468d920..2670e29daf5 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_rp.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_rp.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_rp dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_rt.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_rt.py index 8b7a9257526..08d5e493dec 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_rt.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_rt.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_rt dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_scl.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_scl.py index 00d0d6eb376..420e07157c8 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_scl.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_scl.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_scl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_sdgb.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_sdgb.py index 67932fa7aaf..f48aa45d68b 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_sdgb.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_sdgb.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_sdgb dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_tcl.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_tcl.py index 2df286d3992..c74322718db 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_tcl.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_tcl.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_tcl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_ti.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_ti.py index 682199d79d9..c7d37e401e5 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_ti.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_ti.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_ti dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2023-06-12/faostat_wcad.py b/etl/steps/data/garden/faostat/2023-06-12/faostat_wcad.py index 77ba310b738..cc280289874 100644 --- a/etl/steps/data/garden/faostat/2023-06-12/faostat_wcad.py +++ b/etl/steps/data/garden/faostat/2023-06-12/faostat_wcad.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_wcad dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/additional_variables.py b/etl/steps/data/garden/faostat/2024-03-14/additional_variables.py index da5b1056487..5bd97b9f6df 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/additional_variables.py +++ b/etl/steps/data/garden/faostat/2024-03-14/additional_variables.py @@ -1,6 +1,4 @@ -"""Dataset that combines different variables of other FAOSTAT datasets. - -""" +"""Dataset that combines different variables of other FAOSTAT datasets.""" import numpy as np import owid.catalog.processing as pr diff --git a/etl/steps/data/garden/faostat/2024-03-14/detected_anomalies.py b/etl/steps/data/garden/faostat/2024-03-14/detected_anomalies.py index 2636b4ef8a6..d173e637c28 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/detected_anomalies.py +++ b/etl/steps/data/garden/faostat/2024-03-14/detected_anomalies.py @@ -5,6 +5,7 @@ See documentation of class DataAnomaly below for more details on how anomaly classes are structured. """ + import abc import os from typing import Tuple diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_cahd.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_cahd.py index 048afae3560..02734da3633 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_cahd.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_cahd.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_cahd dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_ei.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_ei.py index 5bd7d23db88..1a013f6e824 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_ei.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_ei.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_ei dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_ek.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_ek.py index c6ec4c862e8..bb2753d1d0a 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_ek.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_ek.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_ek dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_emn.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_emn.py index 5f12637ec70..e2b1eaff5d3 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_emn.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_emn.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_emn dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_esb.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_esb.py index 2a4896b9edf..10fb21b0187 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_esb.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_esb.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_esb dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_fa.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_fa.py index e594773a567..c0c7f77fbfb 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_fa.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_fa.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_fa dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_fbsc.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_fbsc.py index c061395311c..b55ca5f04fa 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_fbsc.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_fbsc.py @@ -15,7 +15,6 @@ """ - import owid.catalog.processing as pr from owid.catalog import Dataset, Table from shared import ( diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_fo.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_fo.py index bba98a5e224..7d4fac7b2d7 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_fo.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_fo.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_fo dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_fs.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_fs.py index a836381fb94..3ea0dd31e98 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_fs.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_fs.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_fs dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_ic.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_ic.py index 5e86234ddde..69e2f85c84c 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_ic.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_ic.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_ic dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_lc.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_lc.py index ab508fd95ad..76ca635f1f8 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_lc.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_lc.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_lc dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_qcl.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_qcl.py index 81fbcf2c011..430ada948a2 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_qcl.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_qcl.py @@ -1,6 +1,5 @@ """FAOSTAT garden step for faostat_qcl dataset.""" - import numpy as np import owid.catalog.processing as pr from owid.catalog import Table diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_qi.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_qi.py index 8c271f07bc2..f671c690199 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_qi.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_qi.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_qi dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_qv.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_qv.py index f564688376e..cb89edeef24 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_qv.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_qv.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_qv dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_rfb.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_rfb.py index 68669b4cbd4..d3300e2ef69 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_rfb.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_rfb.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_rfb dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_rfn.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_rfn.py index 4ebfe341728..20bc8770f77 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_rfn.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_rfn.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_rfn dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_rl.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_rl.py index f43cbe31912..0eea09d6235 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_rl.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_rl.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_rl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_rp.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_rp.py index f15e468d920..2670e29daf5 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_rp.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_rp.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_rp dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_rt.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_rt.py index 8b7a9257526..08d5e493dec 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_rt.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_rt.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_rt dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_scl.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_scl.py index 00d0d6eb376..420e07157c8 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_scl.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_scl.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_scl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_sdgb.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_sdgb.py index 67932fa7aaf..f48aa45d68b 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_sdgb.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_sdgb.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_sdgb dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_tcl.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_tcl.py index 2df286d3992..c74322718db 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_tcl.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_tcl.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_tcl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/faostat/2024-03-14/faostat_ti.py b/etl/steps/data/garden/faostat/2024-03-14/faostat_ti.py index 682199d79d9..c7d37e401e5 100644 --- a/etl/steps/data/garden/faostat/2024-03-14/faostat_ti.py +++ b/etl/steps/data/garden/faostat/2024-03-14/faostat_ti.py @@ -1,2 +1,3 @@ """FAOSTAT garden step for faostat_ti dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/garden/gapminder/2023-03-31/population.py b/etl/steps/data/garden/gapminder/2023-03-31/population.py index 755c9819d92..93ee0ed9ecf 100644 --- a/etl/steps/data/garden/gapminder/2023-03-31/population.py +++ b/etl/steps/data/garden/gapminder/2023-03-31/population.py @@ -4,6 +4,7 @@ More details at https://www.gapminder.org/data/documentation/gd003/. """ + import pandas as pd from owid.catalog import Dataset, Table from structlog import get_logger diff --git a/etl/steps/data/garden/gcp/2023-12-12/global_carbon_budget.py b/etl/steps/data/garden/gcp/2023-12-12/global_carbon_budget.py index 4f946f82ad9..7ec2b8587a7 100644 --- a/etl/steps/data/garden/gcp/2023-12-12/global_carbon_budget.py +++ b/etl/steps/data/garden/gcp/2023-12-12/global_carbon_budget.py @@ -8,6 +8,7 @@ - WorldBank's Income groups, to generate aggregates for different income groups. """ + import numpy as np import owid.catalog.processing as pr from owid.catalog import Dataset, Table @@ -1107,9 +1108,9 @@ def combine_data_and_add_variables( added_variables = tb_co2_with_regions.drop( columns=["country", "year"] + COLUMNS_THAT_MUST_HAVE_DATA ).columns.tolist() - tb_co2_with_regions.loc[ - (tb_co2_with_regions["country"].str.contains(" (GCP)", regex=False)), added_variables - ] = np.nan + tb_co2_with_regions.loc[(tb_co2_with_regions["country"].str.contains(" (GCP)", regex=False)), added_variables] = ( + np.nan + ) # Remove uninformative rows (those that have only data for, say, gdp, but not for variables related to emissions). tb_co2_with_regions = tb_co2_with_regions.dropna(subset=COLUMNS_THAT_MUST_HAVE_DATA, how="all").reset_index( diff --git a/etl/steps/data/garden/gcp/2024-06-20/global_carbon_budget.py b/etl/steps/data/garden/gcp/2024-06-20/global_carbon_budget.py index 082ba1c6d35..07e55586680 100644 --- a/etl/steps/data/garden/gcp/2024-06-20/global_carbon_budget.py +++ b/etl/steps/data/garden/gcp/2024-06-20/global_carbon_budget.py @@ -8,6 +8,7 @@ - WorldBank's Income groups, to generate aggregates for different income groups. """ + import numpy as np import owid.catalog.processing as pr from owid.catalog import Dataset, Table @@ -1117,9 +1118,9 @@ def combine_data_and_add_variables( added_variables = tb_co2_with_regions.drop( columns=["country", "year"] + COLUMNS_THAT_MUST_HAVE_DATA ).columns.tolist() - tb_co2_with_regions.loc[ - (tb_co2_with_regions["country"].str.contains(" (GCP)", regex=False)), added_variables - ] = np.nan + tb_co2_with_regions.loc[(tb_co2_with_regions["country"].str.contains(" (GCP)", regex=False)), added_variables] = ( + np.nan + ) # Remove uninformative rows (those that have only data for, say, gdp, but not for variables related to emissions). tb_co2_with_regions = tb_co2_with_regions.dropna(subset=COLUMNS_THAT_MUST_HAVE_DATA, how="all").reset_index( diff --git a/etl/steps/data/garden/gcp/2024-11-13/global_carbon_budget.py b/etl/steps/data/garden/gcp/2024-11-13/global_carbon_budget.py index 449b89cf2ab..297bc6bdc28 100644 --- a/etl/steps/data/garden/gcp/2024-11-13/global_carbon_budget.py +++ b/etl/steps/data/garden/gcp/2024-11-13/global_carbon_budget.py @@ -8,6 +8,7 @@ - WorldBank's Income groups, to generate aggregates for different income groups. """ + import numpy as np import owid.catalog.processing as pr from owid.catalog import Dataset, Table @@ -1124,9 +1125,9 @@ def combine_data_and_add_variables( added_variables = tb_co2_with_regions.drop( columns=["country", "year"] + COLUMNS_THAT_MUST_HAVE_DATA ).columns.tolist() - tb_co2_with_regions.loc[ - (tb_co2_with_regions["country"].str.contains(" (GCP)", regex=False)), added_variables - ] = np.nan + tb_co2_with_regions.loc[(tb_co2_with_regions["country"].str.contains(" (GCP)", regex=False)), added_variables] = ( + np.nan + ) # Remove uninformative rows (those that have only data for, say, gdp, but not for variables related to emissions). tb_co2_with_regions = tb_co2_with_regions.dropna(subset=COLUMNS_THAT_MUST_HAVE_DATA, how="all").reset_index( diff --git a/etl/steps/data/garden/gcp/2024-11-21/global_carbon_budget.py b/etl/steps/data/garden/gcp/2024-11-21/global_carbon_budget.py index e453ba09dff..7f5fdbc8ce8 100644 --- a/etl/steps/data/garden/gcp/2024-11-21/global_carbon_budget.py +++ b/etl/steps/data/garden/gcp/2024-11-21/global_carbon_budget.py @@ -8,6 +8,7 @@ - WorldBank's Income groups, to generate aggregates for different income groups. """ + import numpy as np import owid.catalog.processing as pr from owid.catalog import Dataset, Table @@ -1125,9 +1126,9 @@ def combine_data_and_add_variables( added_variables = tb_co2_with_regions.drop( columns=["country", "year"] + COLUMNS_THAT_MUST_HAVE_DATA ).columns.tolist() - tb_co2_with_regions.loc[ - (tb_co2_with_regions["country"].str.contains(" (GCP)", regex=False)), added_variables - ] = np.nan + tb_co2_with_regions.loc[(tb_co2_with_regions["country"].str.contains(" (GCP)", regex=False)), added_variables] = ( + np.nan + ) # Remove uninformative rows (those that have only data for, say, gdp, but not for variables related to emissions). tb_co2_with_regions = tb_co2_with_regions.dropna(subset=COLUMNS_THAT_MUST_HAVE_DATA, how="all").reset_index( diff --git a/etl/steps/data/garden/happiness/2023-03-20/happiness.py b/etl/steps/data/garden/happiness/2023-03-20/happiness.py index 3a1c75843de..e5095422300 100644 --- a/etl/steps/data/garden/happiness/2023-03-20/happiness.py +++ b/etl/steps/data/garden/happiness/2023-03-20/happiness.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + import pandas as pd from owid.catalog import Dataset, Table from structlog import get_logger diff --git a/etl/steps/data/garden/health/2023-04-18/shared.py b/etl/steps/data/garden/health/2023-04-18/shared.py index 2701aba1388..dbcd8f2b887 100644 --- a/etl/steps/data/garden/health/2023-04-18/shared.py +++ b/etl/steps/data/garden/health/2023-04-18/shared.py @@ -9,7 +9,6 @@ """ - MAPPING_GENDER_VALUES = { 1: "male", 2: "female", diff --git a/etl/steps/data/garden/health/2023-04-25/shared.py b/etl/steps/data/garden/health/2023-04-25/shared.py index 38923839790..31f3e0fe7e7 100644 --- a/etl/steps/data/garden/health/2023-04-25/shared.py +++ b/etl/steps/data/garden/health/2023-04-25/shared.py @@ -3,7 +3,6 @@ The mappings are created from the metadata dictionary provided by the source (a tab in the excel sheet) """ - # Gender ID mappings MAPPING_GENDER_VALUES = { 1: "male", diff --git a/etl/steps/data/garden/health/2023-08-16/deaths_karlinsky.py b/etl/steps/data/garden/health/2023-08-16/deaths_karlinsky.py index 3d55a0d04ab..b95cda057c4 100644 --- a/etl/steps/data/garden/health/2023-08-16/deaths_karlinsky.py +++ b/etl/steps/data/garden/health/2023-08-16/deaths_karlinsky.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - from owid.catalog import Table from etl.data_helpers import geo diff --git a/etl/steps/data/garden/health/2024-03-21/gmh_countdown.py b/etl/steps/data/garden/health/2024-03-21/gmh_countdown.py index 6978fe4edba..651b4630a03 100644 --- a/etl/steps/data/garden/health/2024-03-21/gmh_countdown.py +++ b/etl/steps/data/garden/health/2024-03-21/gmh_countdown.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - from etl.data_helpers import geo from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/garden/health/2024-04-12/polio_free_countries.py b/etl/steps/data/garden/health/2024-04-12/polio_free_countries.py index 24c21ea0d20..977c62ef052 100644 --- a/etl/steps/data/garden/health/2024-04-12/polio_free_countries.py +++ b/etl/steps/data/garden/health/2024-04-12/polio_free_countries.py @@ -97,9 +97,9 @@ def add_polio_region_certification( # tb_who_region["status"] = "WHO Region certified polio-free" tb = pr.merge(tb, tb_who_region, on=["country", "year"], how="outer") # Set the status for all relevant countries and years - tb.loc[ - tb["country"].isin(country_list) & (tb["year"] >= year_certified_int), "status" - ] = "WHO Region certified polio-free" + tb.loc[tb["country"].isin(country_list) & (tb["year"] >= year_certified_int), "status"] = ( + "WHO Region certified polio-free" + ) return tb diff --git a/etl/steps/data/garden/hmd/2023-09-19/hmd.py b/etl/steps/data/garden/hmd/2023-09-19/hmd.py index bd24d10b6d9..14577b7a894 100644 --- a/etl/steps/data/garden/hmd/2023-09-19/hmd.py +++ b/etl/steps/data/garden/hmd/2023-09-19/hmd.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + from typing import List, cast import numpy as np diff --git a/etl/steps/data/garden/hmd/2024-11-19/hfd.meta.yml b/etl/steps/data/garden/hmd/2024-11-19/hfd.meta.yml index 0e02510f341..7e9d3881034 100644 --- a/etl/steps/data/garden/hmd/2024-11-19/hfd.meta.yml +++ b/etl/steps/data/garden/hmd/2024-11-19/hfd.meta.yml @@ -218,7 +218,39 @@ tables: {definitions.others.title} description_short: |- <% if birth_order == 'total' %> - The average number of children born to women in a specific cohort over their lifetime. + The average number of children born to women in the current cohort over their lifetime. + <%- else %> + The average number of{definitions.others.bo_1} children born to women in a specific cohort over their lifetime. + <%- endif %> + unit: "births per woman" + description_key: + - Represents the completed fertility of a cohort by the end of their reproductive years. + - Useful for comparing fertility across different cohorts. + - Calculated from age-specific fertility rates observed throughout the cohort's reproductive lifespan. + + ccf_plus15y: + title: |- + <% set title = "Completed cohort fertility rate (adjusted +15 years)" %> + {definitions.others.title} + description_short: |- + <% if birth_order == 'total' %> + The average number of children born to women in the 15-years-ago cohort over their lifetime. + <%- else %> + The average number of{definitions.others.bo_1} children born to women in a specific cohort over their lifetime. + <%- endif %> + unit: "births per woman" + description_key: + - Represents the completed fertility of a cohort by the end of their reproductive years. + - Useful for comparing fertility across different cohorts. + - Calculated from age-specific fertility rates observed throughout the cohort's reproductive lifespan. + + ccf_plus30y: + title: |- + <% set title = "Completed cohort fertility rate (adjusted +30 years)" %> + {definitions.others.title} + description_short: |- + <% if birth_order == 'total' %> + The average number of children born to women in the 30-years-ago cohort over their lifetime. <%- else %> The average number of{definitions.others.bo_1} children born to women in a specific cohort over their lifetime. <%- endif %> diff --git a/etl/steps/data/garden/hmd/2024-11-19/hfd.py b/etl/steps/data/garden/hmd/2024-11-19/hfd.py index 2ad25b7d3dd..adc6b83564b 100644 --- a/etl/steps/data/garden/hmd/2024-11-19/hfd.py +++ b/etl/steps/data/garden/hmd/2024-11-19/hfd.py @@ -481,8 +481,26 @@ def run(dest_dir: str) -> None: ) # Quick fix: change birth_order label for PPR tbs = _fix_ppr(tbs) + ## Merge - tb_cohort = consolidate_table_from_list(tbs, cols_index + [col_bo], "cohort") + def add_shifted_to_cohort(tb): + cols_index_all = cols_index + [col_bo] + # Create shifted cohorts + tb_plus15 = tb.copy() + tb_plus15["cohort"] = tb_plus15["cohort"] + 15 + tb_plus30 = tb.copy() + tb_plus30["cohort"] = tb_plus30["cohort"] + 30 + # Merge + tb = tb.merge(tb_plus15[cols_index_all + ["ccf"]], on=cols_index_all, suffixes=["", "_plus15y"], how="outer") + tb = tb.merge(tb_plus30[cols_index_all + ["ccf"]], on=cols_index_all, suffixes=["", "_plus30y"], how="outer") + return tb + + tb_cohort = consolidate_table_from_list( + tbs=tbs, + cols_index_out=cols_index + [col_bo], + short_name="cohort", + fcn=add_shifted_to_cohort, + ) # 3/ Period tables (by age) cols_index = ["country", "year", "age"] diff --git a/etl/steps/data/garden/hmd/2024-12-01/hmd.countries.json b/etl/steps/data/garden/hmd/2024-12-01/hmd.countries.json new file mode 100644 index 00000000000..c88a67232d4 --- /dev/null +++ b/etl/steps/data/garden/hmd/2024-12-01/hmd.countries.json @@ -0,0 +1,48 @@ +{ + "Australia": "Australia", + "Austria": "Austria", + "Belarus": "Belarus", + "Belgium": "Belgium", + "Bulgaria": "Bulgaria", + "Canada": "Canada", + "Chile": "Chile", + "Croatia": "Croatia", + "Czechia": "Czechia", + "Denmark": "Denmark", + "East Germany": "East Germany", + "Estonia": "Estonia", + "Finland": "Finland", + "Germany": "Germany", + "Greece": "Greece", + "Hong Kong": "Hong Kong", + "Hungary": "Hungary", + "Iceland": "Iceland", + "Ireland": "Ireland", + "Japan": "Japan", + "Latvia": "Latvia", + "Lithuania": "Lithuania", + "Luxembourg": "Luxembourg", + "Netherlands": "Netherlands", + "New Zealand": "New Zealand", + "Norway": "Norway", + "Poland": "Poland", + "Portugal": "Portugal", + "Republic of Korea": "South Korea", + "Russia": "Russia", + "Slovenia": "Slovenia", + "Spain": "Spain", + "Sweden": "Sweden", + "Switzerland": "Switzerland", + "Taiwan": "Taiwan", + "Ukraine": "Ukraine", + "United Kingdom": "United Kingdom", + "West Germany": "West Germany", + "England and Wales, Total Population": "England and Wales", + "France, Total Population": "France", + "Israel, Total Population": "Israel", + "Italy ": "Italy", + "Northern Ireland": "Northern Ireland", + "Scotland": "Scotland", + "Slovakia ": "Slovakia", + "The United States of America": "United States" +} diff --git a/etl/steps/data/garden/hmd/2024-12-01/hmd.excluded_countries.json b/etl/steps/data/garden/hmd/2024-12-01/hmd.excluded_countries.json new file mode 100644 index 00000000000..cf4a0297e10 --- /dev/null +++ b/etl/steps/data/garden/hmd/2024-12-01/hmd.excluded_countries.json @@ -0,0 +1,6 @@ +[ + "England and Wales, Civilian National Population", + "France, Civilian Population", + "New Zealand -- Maori", + "New Zealand -- Non-Maori" +] diff --git a/etl/steps/data/garden/hmd/2024-12-01/hmd.meta.yml b/etl/steps/data/garden/hmd/2024-12-01/hmd.meta.yml new file mode 100644 index 00000000000..d846e9b485c --- /dev/null +++ b/etl/steps/data/garden/hmd/2024-12-01/hmd.meta.yml @@ -0,0 +1,404 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + attribution_short: HMD + topic_tags: + - Life Expectancy + + others: + display_name_dim: |- + at << 'birth' if (age == '0') else age >><< ', ' + sex + 's' if (sex != 'total') >>, << type >> + title_public_dim: |- + <% if age != 'total' %>at << age if age != '0' else 'birth'>><% endif %> + global: + life_expectancy: + point_1: |- + <%- if type == "period" -%> + Period life expectancy is a metric that summarizes death rates across all age groups in one particular year. + <%- else -%> + Cohort life expectancy is the average lifespan of a group of people, usually a birth cohort – people born in the same year. + <%- endif -%> + point_2: |- + <%- if type == "period" -%> + <%- if age == '0' -%> + For a given year, it represents the average lifespan for a hypothetical group of people, if they experienced the same age-specific death rates throughout their whole lives as the age-specific death rates seen in that particular year. + <%- else -%> + For a given year, it represents the remaining average lifespan for a hypothetical group of people, if they experienced the same age-specific death rates throughout the rest of their lives as the age-specific death rates seen in that particular year. + <%- endif -%> + <%- else -%> + <%- if age == '0' -%> + It is calculated by tracking individuals from that cohort throughout their lives until death, and calculating their average lifespan. + <%- else -%> + It is calculated by tracking individuals from that cohort throughout the rest of their lives until death, and calculating their average remaining lifespan. + <%- endif -%> + <%- endif -%> + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/dataset/ +dataset: + update_period_days: 365 + description: |- + The Human Mortality Database (HMD) is a collaborative project sponsored by the University of California, Berkeley (in the United States of America) and the Max Planck Institute for Demographic Research (in Germany). + + It provides researchers with comprehensive data on mortality from around 40 countries around the world, which have very high coverage and quality of data at the national level, through vital registration and potentially census data. + + Data is given in terms of period or cohort estimates: + + - **Period data** refers to a snapshot estimated with data at a particular interval. For period life expectancy at birth, this refers to the estimated life expectancy at birth based on a synthetic cohort created using mortality rates across age groups in a given year. + - **Cohort data** refers to estimates of a particular birth cohort. For cohort life expectancy at birth, this refers to the average number of years that people in the birth cohort survived. Cohort data may use birth cohorts that are ‘almost extinct’ rather than entirely extinct. + + 'Interval' refers to the specific age- and time- period of the estimate. An interval can be a one year period for a single-age group, or it can be wider. For example, the life expectancy of a 40 year old in 2019 corresponds to an interval of 1 single-age group in 1 year. The central death rate of 5–9 year olds in 2020 corresponds to an interval of a 5 year age group in 1 year. + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/tables/ +tables: + life_tables: + common: + presentation: + title_variant: << sex + 's, ' if sex != 'total' >><< type + ' tables'>> + topic_tags: + - Life Expectancy + + variables: + central_death_rate: + title: Central death rate + description_short: |- + The death rate, calculated as the number of deaths divided by the average number of people alive during the interval. + description_key: + - "The death rate is measured using the number of person-years lived during the interval." + - "Person-years refers to the combined total time that a group of people has lived. For example, if 10 people each live for 2 years, they collectively contribute 20 person-years." + - "The death rate is slightly different from the 'probability of death' during the interval, because the 'probability of death' metric uses a different denominator: the number of people alive at that age at the start of the interval, while this indicator uses the average number of people alive during the interval." + unit: deaths per 1,000 people + processing_level: minor + description_processing: |- + The original metric is given as a fraction between 0 and 1 (i.e. per-capita). We multiply this by 1,000 to get a per-1,000 people rate. + display: + name: |- + {tables.life_tables.variables.central_death_rate.title} {definitions.others.display_name_dim} + presentation: + title_public: |- + {tables.life_tables.variables.central_death_rate.title} {definitions.others.title_public_dim} + topic_tags: + - Life Expectancy + + probability_of_death: + title: Probability of death + unit: "%" + description_short: |- + The probability of dying in a given interval, among people who survived to the start of that interval. + description_key: + - "For example, the probability of death for a 50 year old in a given year is found by: dividing the number of deaths in 50 year olds that year, by the number of people alive at the age of 50 at the start of the year." + processing_level: minor + description_processing: |- + The original metric is given as a fraction between 0 and 1 (i.e. per-capita). We multiply this by 100 to get a percentage. + display: + name: |- + {tables.life_tables.variables.probability_of_death.title} {definitions.others.display_name_dim} + presentation: + title_public: |- + {tables.life_tables.variables.probability_of_death.title} {definitions.others.title_public_dim} + topic_tags: + - Life Expectancy + + average_survival_length: + title: Average survival length + short_unit: years + unit: years + description_short: Average length of survival between ages x and x+n for persons dying in the interval. + display: + name: |- + {tables.life_tables.variables.average_survival_length.title} {definitions.others.display_name_dim} + presentation: + title_public: |- + {tables.life_tables.variables.average_survival_length.title} {definitions.others.title_public_dim} + + number_survivors: + title: Number of survivors + unit: survivors + description_short: Number of survivors at a given age, assuming survivors at 0 years old is 100,000. + display: + name: |- + {tables.life_tables.variables.number_survivors.title} {definitions.others.display_name_dim} + presentation: + title_public: |- + {tables.life_tables.variables.number_survivors.title} {definitions.others.title_public_dim} + + number_deaths: + title: Number of deaths + short_unit: deaths + unit: deaths + description_short: Number of deaths between ages x and x+n. + display: + name: |- + {tables.life_tables.variables.number_deaths.title} {definitions.others.display_name_dim} + presentation: + title_public: |- + {tables.life_tables.variables.number_deaths.title} {definitions.others.title_public_dim} + topic_tags: + - Life Expectancy + + number_person_years_lived: + title: Number of person-years lived + unit: person-years + description_short: Number of person-years lived between ages x and x+n. + display: + name: |- + {tables.life_tables.variables.number_person_years_lived.title} {definitions.others.display_name_dim} + presentation: + title_public: |- + {tables.life_tables.variables.number_person_years_lived.title} {definitions.others.title_public_dim} + + number_person_years_remaining: + title: Number of person-years remaining + unit: person-years + description_short: Number of person-years remaining after a given age. + display: + name: |- + {tables.life_tables.variables.number_person_years_remaining.title} {definitions.others.display_name_dim} + presentation: + title_public: |- + {tables.life_tables.variables.number_person_years_remaining.title} {definitions.others.title_public_dim} + + life_expectancy: + title: Life expectancy + short_unit: years + unit: years + description_short: |- + <%- if age == '0' -%> + <%- if sex == 'total' -%> + The << type >> life expectancy at birth, in a given year. + <%- else -%> + The << type >> life expectancy at birth among << sex + 's' >>, in a given year. + <%- endif -%> + <%- else -%> + <%- if sex == 'total' -%> + The remaining << type >> life expectancy at age << age >>, in a given year. + <%- else -%> + The remaining << type >> life expectancy at age << age >> among << sex + 's' >>, in a given year. + <%- endif -%> + <%- endif -%> + description_key: + - |- + {definitions.global.life_expectancy.point_1} + - |- + {definitions.global.life_expectancy.point_2} + - |- + <%- if age != '0' -%> + <%- if type == "period" -%> + This shows the remaining period life expectancy among people who have already reached the age << age >>, using death rates from their age group and older age groups. + <%- else -%> + This shows the remaining cohort life expectancy of people who have reached the age << age >>. + <%- endif -%> + <%- endif -%> + display: + numDecimalPlaces: 1 + name: |- + {tables.life_tables.variables.life_expectancy.title} {definitions.others.display_name_dim} + presentation: + title_public: |- + {tables.life_tables.variables.life_expectancy.title} {definitions.others.title_public_dim} + + exposures: + common: + presentation: + title_variant: << sex + 's, ' if sex != 'total' >><< type + ' tables'>> + topic_tags: + - Life Expectancy + + variables: + exposure: + title: Exposure-to-risk + unit: person-years + description_short: The total number of person-years lived within a given interval. + description_key: + - It is equivalent to the average number of people living in that age group during the period. + description_from_producer: |- + Estimates of the population exposed to the risk of death during some age-time interval are based on annual (January 1st) population estimates, with small corrections that reflect the timing of deaths during the interval. Period exposure estimations are based on assumptions of uniformity in the distribution of events except when historical monthly birth data are available. + display: + name: |- + {tables.exposures.variables.exposure.title} {definitions.others.display_name_dim} + presentation: + title_public: |- + {tables.exposures.variables.exposure.title} {definitions.others.title_public_dim} + + deaths: + common: + presentation: + topic_tags: + - Global Health + title_variant: << sex + 's, ' if sex != 'total' >> + + variables: + deaths: + title: Number of deaths + unit: deaths + description_short: |- + <% if sex == 'total' %> + The total number of deaths at age << age >> in a given year. + <%- else %> + The total number of << sex >> deaths at age << age >> in a given year. + <%- endif %> + display: + name: |- + {tables.deaths.variables.deaths.title} at << 'birth' if (age == '0') else age >><< ', ' + sex + 's' if (sex != 'total') >> + presentation: + title_public: |- + {tables.deaths.variables.deaths.title} {definitions.others.title_public_dim} + + population: + common: + presentation: + topic_tags: + - Population Growth + + variables: + population: + title: Population + unit: people + description_short: |- + <% if age == 'total' %> + <%- if sex == 'total' %> + The total number of people living in a country. + <%- else %> + The total number of << sex + 's' >> living in a country. + <%- endif %> + <%- else %> + <% if sex == 'total' %> + The total number of people aged << age >> living in a country. + <%- else %> + The total number of << sex + 's' >> aged << age >> living in a country. + <%- endif %> + <%- endif %> + description_processing: |- + From HMD Notes: For populations with territorial changes, two sets of population estimates are given for years in which a territorial change occurred. The first set of estimates (identified as year "19xx-") refers to the population just before the territorial change, whereas the second set (identified as year "19xx+") refers to the population just after the change. For example, in France, the data for "1914-" cover the previous territory (i.e., as of December 31, 1913), whereas the data for "1914+" reflect the territorial boundaries as of January 1, 1914. + + We have used the "19xx+" population estimates for the year of the territorial change. + display: + name: |- + {tables.population.variables.population.title}<< 'aged ' + age if (age != 'total') >><< ', ' + sex + 's' if (sex != 'total') >> + presentation: + title_public: |- + {tables.population.variables.population.title} {definitions.others.title_public_dim} + title_variant: << sex + 's, ' if sex != 'total' >> + + births: + common: + presentation: + topic_tags: + - Fertility Rate + title_variant: << sex + 's, ' if sex != 'total' >> + + variables: + births: + title: Births + unit: births + description_short: |- + <% if sex == 'total' %> + The total number of births in a given year. + <%- else %> + The total number of << sex >> births in a given year. + <%- endif %> + display: + name: |- + Births, sex: << sex >> + presentation: + title_public: |- + {tables.births.variables.births.title}, + <%- if sex == 'total' %> + total + <%- else %> + << sex >>s + <%- endif %> + birth_rate: + title: Birth rate + unit: births per 1,000 people + description_short: |- + <% if sex == 'total' %> + The total number of births per 1,000 people in a given year. + <%- else %> + The total number of << sex >> births per 1,000 in a given year. + <%- endif %> + display: + name: |- + Birth rate, sex: << sex >> + presentation: + title_public: |- + {tables.births.variables.birth_rate.title}, + <%- if sex == 'total' %> + total + <%- else %> + << sex >>s + <%- endif %> + + diff_ratios: + common: + presentation: + topic_tags: + - Life Expectancy + + variables: + central_death_rate_mf_ratio: + title: Central death rate ratio (m/f) + unit: "" + description_short: |- + The ratio of the << type >> central death rate (males to females) at age << age >>. + processing_level: major + display: + name: |- + Central death rate (male-to-female ratio) at << 'birth' if (age == '0') else age >>, << type >> + presentation: + title_public: Central death rate {definitions.others.title_public_dim} + title_variant: |- + male-to-female ratio, << type >> tables + topic_tags: + - Life Expectancy + - Gender Ratio + + life_expectancy_fm_diff: + title: Life expectancy difference (f-m) + short_unit: years + unit: years + description_short: |- + The difference in the << type >> life expectancy (females - males) at age << age >>. + processing_level: major + description_key: + - Higher values indicate longer life expectancy among females than males. + - |- + {definitions.global.life_expectancy.point_1} + - |- + {definitions.global.life_expectancy.point_2} + display: + numDecimalPlaces: 1 + name: |- + Life expectancy (female-male difference) at << 'birth' if (age == '0') else age >>, << type >> + presentation: + title_public: Life expectancy at << age if age != '0' else 'birth'>> + title_variant: female-male difference, << type >> tables + topic_tags: + - Life Expectancy + - Gender Ratio + + life_expectancy_fm_ratio: + title: Life expectancy ratio (f/m) + unit: "" + short_unit: "" + description_short: |- + The ratio of the << type >> life expectancy (females to males) at age << age >>. + processing_level: major + description_key: + - Higher values indicate longer life expectancy among females than males. + - |- + {definitions.global.life_expectancy.point_1} + - |- + {definitions.global.life_expectancy.point_2} + display: + numDecimalPlaces: 1 + name: |- + Life expectancy (female-to-male ratio) at << 'birth' if (age == '0') else age >>, << type >> + presentation: + title_public: Life expectancy at << age if age != '0' else 'birth'>> + title_variant: female-to-male ratio, << type >> tables + topic_tags: + - Life Expectancy + - Gender Ratio diff --git a/etl/steps/data/garden/hmd/2024-12-01/hmd.py b/etl/steps/data/garden/hmd/2024-12-01/hmd.py new file mode 100644 index 00000000000..471502c0299 --- /dev/null +++ b/etl/steps/data/garden/hmd/2024-12-01/hmd.py @@ -0,0 +1,240 @@ +"""Load a meadow dataset and create a garden dataset.""" + +import numpy as np +from owid.catalog import Table +from owid.catalog import processing as pr + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("hmd") + + # Read table from meadow dataset. + paths.log.info("reading tables") + tb_lt = ds_meadow.read("life_tables") + tb_exp = ds_meadow.read("exposures") + tb_mort = ds_meadow.read("deaths") + tb_pop = ds_meadow.read("population") + tb_births = ds_meadow.read("births") + + # Drop NaNs + tb_exp = tb_exp.dropna(subset="exposure") + tb_births = tb_births.dropna(subset="births") + + # + # Process data. + # + paths.log.info("processing tables") + + # 1/ Life tables + def _sanity_check_lt(tb): + summary = tb.groupby(["country", "year", "sex", "type", "age"], as_index=False).size().sort_values("size") + row_dups = summary.loc[summary["size"] != 1] + assert row_dups.shape[0] <= 19, "Found duplicated rows in life tables!" + assert (row_dups["country"].unique() == "Switzerland").all() & ( + row_dups["year"] <= 1931 + ).all(), "Unexpected duplicates in life tables!" + + flag = ( + (tb_lt["country"] == "Switzerland") + & (tb_lt["age"] == "110+") + & (tb_lt["type"] == "cohort") + & (tb_lt["sex"] == "male") + & (tb_lt["year"] <= 1931) + & (tb_lt["year"] >= 1913) + ) + tb = tb.loc[~flag] + + return tb + + tb_lt = process_table( + tb=tb_lt, + col_index=["country", "year", "sex", "age", "type"], + sex_expected={"females", "males", "total"}, + callback_post=_sanity_check_lt, + ) + # Scale central death rates + tb_lt["central_death_rate"] = tb_lt["central_death_rate"] * 1_000 + tb_lt["probability_of_death"] = tb_lt["probability_of_death"] * 100 + + # 2/ Exposures + tb_exp = process_table( + tb=tb_exp, + col_index=["country", "year", "sex", "age", "type"], + ) + + # 3/ Mortality + tb_mort = process_table( + tb=tb_mort, + col_index=["country", "year", "sex", "age", "type"], + ) + assert set(tb_mort["type"].unique()) == {"period"}, "Unexpected values in column 'type' in mortality tables!" + tb_mort = tb_mort.drop(columns="type") + + # 4/ Population + tb_pop = process_table( + tb=tb_pop, + col_index=["country", "year", "sex", "age"], + ) + tb_pop = add_total_population(tb_pop) + + # 5/ Births + tb_births = process_table( + tb=tb_births, + col_index=["country", "year", "sex"], + ) + + def add_birth_rate(tb_pop, tb_births): + tb_pop_agg = tb_pop[tb_pop["age"] == "total"].drop(columns="age") + tb_births = tb_births.merge(tb_pop_agg, on=["country", "year", "sex"], how="left") + tb_births["birth_rate"] = tb_births["births"] / tb_births["population"] * 1_000 + tb_births["birth_rate"] = tb_births["birth_rate"].replace([np.inf, -np.inf], np.nan) + tb_births = tb_births.drop(columns=["population"]) + return tb_births + + tb_births = add_birth_rate(tb_pop, tb_births) + + # 6/ Create table with differences and ratios + tb_ratios = make_table_diffs_ratios(tb_lt) + + # Create list with tables + paths.log.info("saving tables") + tables = [ + tb_lt.format(["country", "year", "sex", "age", "type"]), + tb_exp.format(["country", "year", "sex", "age", "type"]), + tb_mort.format(["country", "year", "sex", "age"]), + tb_pop.format(["country", "year", "sex", "age"]), + tb_births.format(["country", "year", "sex"]), + tb_ratios.format(["country", "year", "age", "type"], short_name="diff_ratios"), + ] + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, + tables=tables, + check_variables_metadata=True, + ) + + # Save changes in the new garden dataset. + ds_garden.save() + + +def process_table(tb, col_index, sex_expected=None, callback_post=None): + """Reshape a table. + + Input table has column `format`, which is sort-of redundant. This function ensures we can safely drop it (i.e. no duplicate rows). + + Additionally, it standardizes the dimension values. + """ + paths.log.info(f"processing table {tb.m.short_name}") + + if sex_expected is None: + sex_expected = {"female", "male", "total"} + + # Standardize dimension values + tb = standardize_sex_cat_names(tb, sex_expected) + + # Drop duplicate rows + tb = tb.sort_values("format").drop_duplicates(subset=[col for col in tb.columns if col != "format"], keep="first") + + # Check no duplicates + if callback_post is not None: + tb = callback_post(tb) + else: + summary = tb.groupby(col_index, as_index=False).size().sort_values("size") + row_dups = summary.loc[summary["size"] != 1] + assert row_dups.empty, "Found duplicated rows in life tables!" + + # Final dropping o f columns + tb = tb.drop(columns="format") + + # Country name standardization + tb = geo.harmonize_countries( + df=tb, + countries_file=paths.country_mapping_path, + excluded_countries_file=paths.excluded_countries_path, + ) + + # Make year column integer + tb["year"] = tb["year"].astype(int) + + return tb + + +def standardize_sex_cat_names(tb, sex_expected): + # Define expected sex categories + sex_expected = {s.lower() for s in sex_expected} + + # Set sex categories to lowercase + tb["sex"] = tb["sex"].str.lower() + + # Sanity check categories + sex_found = set(tb["sex"].unique()) + assert sex_found == sex_expected, f"Unexpected sex categories! Found {sex_found} but expected {sex_expected}" + + # Rename + tb["sex"] = tb["sex"].replace({"females": "female", "males": "male"}) + + return tb + + +def add_total_population(tb_pop): + flag = tb_pop["age"].str.match(r"^(\d{1,3}|\d{3}\+)$") + tb_pop_total = tb_pop[flag] + tb_pop_total = tb_pop_total.groupby(["country", "year", "sex"], as_index=False)["population"].sum() + tb_pop_total["age"] = "total" + tb_pop = pr.concat([tb_pop, tb_pop_total], ignore_index=True) + return tb_pop + + +def make_table_diffs_ratios(tb: Table) -> Table: + """Create table with metric differences and ratios. + + Currently, we estimate: + + - female - male: Life expectancy + - male/female: Life Expectancy, Central Death Rate + """ + # Pivot & obtain differences and ratios + cols_index = ["country", "year", "age", "type"] + tb_new = ( + tb.pivot_table( + index=cols_index, + columns="sex", + values=["life_expectancy", "central_death_rate"], + ) + .assign( + life_expectancy_fm_diff=lambda df: df[("life_expectancy", "female")] - df[("life_expectancy", "male")], + life_expectancy_fm_ratio=lambda df: df[("life_expectancy", "female")] / df[("life_expectancy", "male")], + central_death_rate_mf_ratio=lambda df: df[("central_death_rate", "male")] + / df[("central_death_rate", "female")], + ) + .reset_index() + ) + + # Keep relevant columns + cols = [col for col in tb_new.columns if col[1] == ""] + tb_new = tb_new.loc[:, cols] + + # Rename columns + tb_new.columns = [col[0] for col in tb_new.columns] + + # Add metadata back + for col in tb_new.columns: + if col not in cols_index: + tb_new[col].metadata.origins = tb["life_expectancy"].m.origins.copy() + tb_new[col] = tb_new[col].replace([np.inf, -np.inf], np.nan) + + return tb_new diff --git a/etl/steps/data/garden/hmd/2024-12-03/hmd_country.countries.json b/etl/steps/data/garden/hmd/2024-12-03/hmd_country.countries.json new file mode 100644 index 00000000000..26b62193714 --- /dev/null +++ b/etl/steps/data/garden/hmd/2024-12-03/hmd_country.countries.json @@ -0,0 +1,47 @@ +{ + "AUS": "Australia", + "AUT": "Austria", + "BEL": "Belgium", + "BGR": "Bulgaria", + "BLR": "Belarus", + "CAN": "Canada", + "CHE": "Switzerland", + "CHL": "Chile", + "CZE": "Czechia", + "DNK": "Denmark", + "ESP": "Spain", + "EST": "Estonia", + "FIN": "Finland", + "GRC": "Greece", + "HKG": "Hong Kong", + "HRV": "Croatia", + "HUN": "Hungary", + "IRL": "Ireland", + "ISL": "Iceland", + "ISR": "Israel", + "ITA": "Italy", + "JPN": "Japan", + "KOR": "South Korea", + "LTU": "Lithuania", + "LUX": "Luxembourg", + "LVA": "Latvia", + "NLD": "Netherlands", + "NOR": "Norway", + "POL": "Poland", + "PRT": "Portugal", + "RUS": "Russia", + "SVK": "Slovakia", + "SVN": "Slovenia", + "SWE": "Sweden", + "UKR": "Ukraine", + "USA": "United States", + "DEUTE": "East Germany", + "DEUTNP": "Germany", + "DEUTW": "West Germany", + "FRATNP": "France", + "GBRTENW": "England and Wales", + "GBR_NIR": "Northern Ireland", + "GBR_NP": "United Kingdom", + "GBR_SCO": "Scotland", + "NZL_NP": "New Zealand" +} diff --git a/etl/steps/data/garden/hmd/2024-12-03/hmd_country.excluded_countries.json b/etl/steps/data/garden/hmd/2024-12-03/hmd_country.excluded_countries.json new file mode 100644 index 00000000000..3a7f14126b5 --- /dev/null +++ b/etl/steps/data/garden/hmd/2024-12-03/hmd_country.excluded_countries.json @@ -0,0 +1,4 @@ +[ + "FRACNP", + "GBRCENW" +] diff --git a/etl/steps/data/garden/hmd/2024-12-03/hmd_country.meta.yml b/etl/steps/data/garden/hmd/2024-12-03/hmd_country.meta.yml new file mode 100644 index 00000000000..96e7e2a8e04 --- /dev/null +++ b/etl/steps/data/garden/hmd/2024-12-03/hmd_country.meta.yml @@ -0,0 +1,111 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Fertility Rate + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + title: Birth rate by month (HMD) + update_period_days: 365 + +tables: + birth_rate: + variables: + birth_rate: + title: Birth rate, on a monthly basis + unit: births per 1,000 people + description_short: |- + The total number of births per 1,000 people in a given month. + display: + name: |- + Birth rate + + birth_rate_per_day: + title: Birth rate per day, on a monthly basis + unit: births per million people + description_short: |- + The average daily number of births, per million people, calculated monthly. + display: + name: |- + Birth rate, per day + + birth_rate_lead_9months: + title: Birth rate, on a monthly basis, by estimated month of conception + unit: births per 1,000 people + description_short: |- + The total number of births per 1,000 people in a given month. + display: + name: |- + Birth rate + + birth_rate_per_day_lead_9months: + title: Birth rate per day, on a monthly basis, by estimated month of conception + unit: births per 1,000 people + description_short: |- + The average daily number of births, per million people, calculated monthly. + display: + name: |- + Birth rate, per day + + birth_rate_month: + variables: + birth_rate: + title: Birth rate, in << month >> + unit: births per 1,000 people + description_short: |- + The total number of births per 1,000 people in <>. + display: + name: |- + Birth rate + + birth_rate_per_day: + title: Birth rate per day, in << month >> + unit: births per million people + description_short: |- + The average daily number of births, per million people, calculated for <>. + display: + name: |- + Birth rate, per day + + birth_rate_month_max: + variables: + month_max: + title: Month ordinal with peak daily birth rate + unit: "" + description_short: |- + Number corresponding to the month with the highest daily birth rate. + month_max_name: + title: Month name with peak daily birth rate + unit: "" + description_short: |- + Month with the highest daily birth rate. + birth_rate_per_day_max: + title: Peak birth rate per day, on a monthly basis + unit: births per million people + description_short: |- + The highest average daily number of births, per million people, recorded in the given year. + display: + name: |- + Maximum birth rate, per day + + month_max_lead_9months: + title: Month ordinal with peak daily birth rate in 9 months + unit: "" + description_short: |- + Number corresponding to the month with the highest daily birth rate. + month_max_name_lead_9months: + title: Month name with peak daily birth rate in 9 months + unit: "" + description_short: |- + Month with the highest daily birth rate. + birth_rate_per_day_max_lead_9months: + title: Peak birth rate per day, on a monthly basis, in 9 months + unit: births per million people + description_short: |- + The highest average daily number of births, per million people, recorded in the given year. + display: + name: |- + Maximum birth rate, per day diff --git a/etl/steps/data/garden/hmd/2024-12-03/hmd_country.py b/etl/steps/data/garden/hmd/2024-12-03/hmd_country.py new file mode 100644 index 00000000000..87a46550a1b --- /dev/null +++ b/etl/steps/data/garden/hmd/2024-12-03/hmd_country.py @@ -0,0 +1,187 @@ +"""Load a meadow dataset and create a garden dataset.""" + +import calendar + +import numpy as np +import pandas as pd + +from etl.data_helpers import geo +from etl.data_helpers.misc import interpolate_table +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("hmd_country") + ds_hmd = paths.load_dataset("hmd") + + # Read table from meadow dataset. + tb = ds_meadow.read("monthly") + tb_pop = ds_hmd.read("population") + + # + # Process data. + # + tb = make_main_table(tb, tb_pop) + tb_month_long, tb_month_dimensions, tb_month_max = make_tables(tb) + + tables = [ + tb_month_long.format(["country", "date"], short_name="birth_rate"), + tb_month_dimensions.format(["country", "year", "month"], short_name="birth_rate_month"), + tb_month_max.format(["country", "year"], short_name="birth_rate_month_max"), + ] + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, + tables=tables, + check_variables_metadata=True, + default_metadata=ds_meadow.metadata, + ) + + # Save changes in the new garden dataset. + ds_garden.save() + + +def make_main_table(tb, tb_pop): + ## Discard unknown/total values + tb = clean_table(tb) + + # Add population to monthly birth data table + tb = add_population_column(tb, tb_pop) + + # Estimate metrics + tb = estimate_metrics(tb) + + # Fix date (use last day instead of first) + tb["date"] = tb["date"] + pd.to_timedelta(tb["days_in_month"] - 1, unit="D") + + # Sort rows + tb = tb.sort_values(["country", "date", "date"]) + + return tb + + +def make_tables(tb): + # Classic time-series, with date-values + tb_long = tb[["country", "date", "birth_rate", "birth_rate_per_day"]] + ## Add 9-month lead + tb_long["date_lead"] = (tb_long[["date"]] - pd.DateOffset(months=9) + pd.offsets.MonthEnd(0)).squeeze() + cols = [col for col in tb_long.columns if col != "date_lead"] + cols_lead = [col for col in tb_long.columns if col != "date"] + tb_long = tb_long[cols].merge( + tb_long[cols_lead], + left_on=["country", "date"], + right_on=["country", "date_lead"], + suffixes=("", "_lead_9months"), + how="outer", + ) + tb_long["date"] = tb_long["date"].fillna(tb_long["date_lead"]) + tb_long = tb_long.drop(columns="date_lead") + + # Month as a dimension + tb_dimensions = tb[["country", "year", "month", "birth_rate", "birth_rate_per_day"]] + tb_dimensions["month"] = tb_dimensions["month"].apply(lambda x: calendar.month_name[x]) + + # For each year, ID of the month with highest birth rate per day + def find_peak_month(tb): + tb = tb.loc[ + tb.groupby(["country", "year"])["birth_rate_per_day"].idxmax(), + ["country", "year", "month", "birth_rate_per_day"], + ].rename(columns={"month": "month_max", "birth_rate_per_day": "birth_rate_per_day_max"}) + tb["month_max_name"] = tb["month_max"].apply(lambda x: calendar.month_name[x]) + + return tb + + tb_month_max = find_peak_month(tb) + # Get: "for a given year, what was the month that lead to highest month-birth rate in +9months" + tb_pre9m = tb.copy() + tb_pre9m["date"] = (tb_pre9m[["date"]] - pd.DateOffset(months=9) + pd.offsets.MonthEnd(0)).squeeze() + tb_pre9m["year"] = tb_pre9m["date"].dt.year + tb_pre9m["year"] = tb_pre9m["year"].copy_metadata(tb["year"]) + tb_pre9m["month"] = tb_pre9m["date"].dt.month + tb_pre9m["month"] = tb_pre9m["month"].copy_metadata(tb["month"]) + + tb_pre9m = find_peak_month(tb_pre9m) + # Merge + tb_month_max = tb_month_max.merge(tb_pre9m, on=["country", "year"], how="outer", suffixes=("", "_lead_9months")) + return tb_long, tb_dimensions, tb_month_max + + +def clean_table(tb): + """Filter rows, harmonize country names, add date column.""" + # Filter unwanted month categories, set dtype + tb = tb.loc[~tb["month"].isin(["TOT", "UNK"])] + tb["month"] = tb["month"].astype("Int64") + ## Create date column. TODO: check what day of the month to assign + tb["date"] = pd.to_datetime(tb[["year", "month"]].assign(day=1)) + # Harmonize country names + tb = geo.harmonize_countries( + df=tb, + countries_file=paths.country_mapping_path, + excluded_countries_file=paths.excluded_countries_path, + warn_on_unknown_excluded_countries=False, + ) + + return tb + + +def add_population_column(tb, tb_pop): + """Add population column to main table for each date.""" + # Prepare population table + tb_pop = _prepare_population_table(tb_pop) + # Merge population table with main table + tb = tb.merge(tb_pop, on=["country", "date"], how="outer") + tb = tb.sort_values(["country", "date"]) + # Interpolate to get monthly population estimates + tb_ = interpolate_table( + tb[["country", "date", "population"]], + entity_col="country", + time_col="date", + time_mode="none", + ) + tb = tb.drop(columns="population").merge(tb_, on=["country", "date"], how="left") + # Drop unused rows + tb = tb.dropna(subset=["year"]) + return tb + + +def _prepare_population_table(tb): + """Prepare population table to merge with main table. + + Original table is given in years, but we need it in days! We use linear interpolation for that. + """ + tb_aux = tb.loc[(tb["age"] == "total") & (tb["sex"] == "total"), ["country", "year", "population"]] + ## Assign a day to population. TODO: Check if this is true + tb_aux["date"] = pd.to_datetime(tb_aux["year"].astype(str) + "-01-01") + tb_aux = tb_aux.drop(columns="year") + + return tb_aux + + +def estimate_metrics(tb): + """Estimate metrics: birth rate and birth rate per day.""" + # Get days in month + tb["days_in_month"] = tb.apply(lambda row: calendar.monthrange(row["year"], row["month"])[1], axis=1) + # Estimate rates + tb["birth_rate"] = tb["births"] / tb["population"] * 1_000 + tb["birth_rate_per_day"] = tb["birth_rate"] / tb["days_in_month"] * 1_000 + # Check + assert tb[["birth_rate", "birth_rate_per_day"]].notna().all().all() + # Replace INF values with NAs + tb[["birth_rate", "birth_rate_per_day"]] = tb[["birth_rate", "birth_rate_per_day"]].replace( + [np.inf, -np.inf], pd.NA + ) + # Drop NAs + tb = tb.dropna(subset=["birth_rate", "birth_rate_per_day"]) + + return tb diff --git a/etl/steps/data/garden/hyde/2024-01-02/all_indicators.py b/etl/steps/data/garden/hyde/2024-01-02/all_indicators.py index 7b6f521f726..60ccc6d03db 100644 --- a/etl/steps/data/garden/hyde/2024-01-02/all_indicators.py +++ b/etl/steps/data/garden/hyde/2024-01-02/all_indicators.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + from etl.data_helpers import geo from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/garden/iea/2024-07-04/critical_minerals.py b/etl/steps/data/garden/iea/2024-07-04/critical_minerals.py index e3e656b7186..1efba2e7d93 100644 --- a/etl/steps/data/garden/iea/2024-07-04/critical_minerals.py +++ b/etl/steps/data/garden/iea/2024-07-04/critical_minerals.py @@ -523,12 +523,12 @@ def harmonize_minerals_and_processes(tb_demand: Table, tb_supply: Table) -> Tupl # * For graphite, supply data is divided in "mining (natural)" and "battery grade". For consistency, we can rename them "mining" and "refining" and add a footnote. tb_supply = tb_supply.astype({"process": "string", "mineral": "string"}).copy() tb_supply.loc[(tb_supply["mineral"] == "Lithium") & (tb_supply["process"] == "Chemicals"), "process"] = "Refining" - tb_supply.loc[ - (tb_supply["mineral"] == "Graphite") & (tb_supply["process"] == "Battery grade"), "process" - ] = "Refining" - tb_supply.loc[ - (tb_supply["mineral"] == "Graphite") & (tb_supply["process"] == "Mining (natural)"), "process" - ] = "Mining" + tb_supply.loc[(tb_supply["mineral"] == "Graphite") & (tb_supply["process"] == "Battery grade"), "process"] = ( + "Refining" + ) + tb_supply.loc[(tb_supply["mineral"] == "Graphite") & (tb_supply["process"] == "Mining (natural)"), "process"] = ( + "Mining" + ) # Rename a few things, for consistency with the minerals explorer. for table in [tb_demand, tb_supply]: diff --git a/etl/steps/data/garden/iea/2024-11-20/fossil_fuel_subsidies.countries.json b/etl/steps/data/garden/iea/2024-11-20/fossil_fuel_subsidies.countries.json new file mode 100644 index 00000000000..37c3e86047f --- /dev/null +++ b/etl/steps/data/garden/iea/2024-11-20/fossil_fuel_subsidies.countries.json @@ -0,0 +1,51 @@ +{ + "Algeria": "Algeria", + "Angola": "Angola", + "Argentina": "Argentina", + "Austria": "Austria", + "Azerbaijan": "Azerbaijan", + "Bahrain": "Bahrain", + "Bangladesh": "Bangladesh", + "Bolivia": "Bolivia", + "Brunei": "Brunei", + "China": "China", + "Colombia": "Colombia", + "Croatia": "Croatia", + "Ecuador": "Ecuador", + "Egypt": "Egypt", + "France": "France", + "Gabon": "Gabon", + "Ghana": "Ghana", + "Hungary": "Hungary", + "India": "India", + "Indonesia": "Indonesia", + "Iran": "Iran", + "Iraq": "Iraq", + "Kazakhstan": "Kazakhstan", + "Kuwait": "Kuwait", + "Libya": "Libya", + "Malaysia": "Malaysia", + "Mexico": "Mexico", + "Nigeria": "Nigeria", + "Oman": "Oman", + "Pakistan": "Pakistan", + "Poland": "Poland", + "Qatar": "Qatar", + "Russia": "Russia", + "Slovak Republic": "Slovakia", + "Thailand": "Thailand", + "Turkmenistan": "Turkmenistan", + "UAE": "United Arab Emirates", + "Ukraine": "Ukraine", + "United Kingdom": "United Kingdom", + "Uzbekistan": "Uzbekistan", + "Venezuela": "Venezuela", + "Vietnam": "Vietnam", + "World": "World", + "ElSalvador": "El Salvador", + "SaudiArabia": "Saudi Arabia", + "SouthAfrica": "South Africa", + "SriLanka": "Sri Lanka", + "Taipei": "Taiwan", + "TrinidadandTobago": "Trinidad and Tobago" +} \ No newline at end of file diff --git a/etl/steps/data/garden/iea/2024-11-20/fossil_fuel_subsidies.meta.yml b/etl/steps/data/garden/iea/2024-11-20/fossil_fuel_subsidies.meta.yml new file mode 100644 index 00000000000..772f0f01622 --- /dev/null +++ b/etl/steps/data/garden/iea/2024-11-20/fossil_fuel_subsidies.meta.yml @@ -0,0 +1,45 @@ +definitions: + common: + presentation: + topic_tags: + - Energy + - Fossil Fuels + grapher_config: + note: This data is expressed in constant {dollar_year} US$. + processing_level: minor + unit: constant {dollar_year} US$ + short_unit: "$" + description_short: |- + This data is expressed in US dollars. It is adjusted for inflation but does not account for differences in living costs between countries. + +dataset: + update_period_days: 365 + +tables: + fossil_fuel_subsidies: + variables: + coal_subsidy: + title: Subsidies to coal + electricity_subsidy: + title: Subsidies to electricity + gas_subsidy: + title: Subsidies to gas + oil_subsidy: + title: Subsidies to oil + total_subsidy: + title: Total subsidies + transport_oil_subsidy: + title: Subsidies to oil in transport + subsidization_rate: + title: Subsidization rate + unit: "%" + short_unit: "%" + subsidy_per_capita: + title: Subsidy per capita + unit: constant {dollar_year} US$ per person + short_unit: "$/person" + subsidy_as_share_of_gdp: + title: Subsidy as share of GDP + unit: "%" + short_unit: "%" + diff --git a/etl/steps/data/garden/iea/2024-11-20/fossil_fuel_subsidies.py b/etl/steps/data/garden/iea/2024-11-20/fossil_fuel_subsidies.py new file mode 100644 index 00000000000..986ad16ad85 --- /dev/null +++ b/etl/steps/data/garden/iea/2024-11-20/fossil_fuel_subsidies.py @@ -0,0 +1,58 @@ +"""Load a meadow dataset and create a garden dataset.""" + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +# Assumed USD year. +DOLLAR_YEAR = 2023 + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("fossil_fuel_subsidies") + + # Read tables from meadow dataset. + tb = ds_meadow.read("fossil_fuel_subsidies") + tb_indicators = ds_meadow.read("fossil_fuel_subsidies_indicators") + tb_transport = ds_meadow.read("fossil_fuel_subsidies_transport_oil") + + # + # Process data. + # + # Convert units from millions of dollars to dollars. + tb["subsidy"] *= 1e6 + + # Transpose table. + tb = tb.pivot(index=["country", "year"], columns="product", values="subsidy", join_column_levels_with="_") + + # Rename conveniently. + tb = tb.rename( + columns={column: f"{column}_subsidy" for column in tb.drop(columns=["country", "year"]).columns}, errors="raise" + ) + + # Harmonize country names. + tb = geo.harmonize_countries(df=tb, countries_file=paths.country_mapping_path) + + # Include additional indicators from the other tables. + tb = tb.merge(tb_indicators, on=["country", "year"], how="outer") + tb = tb.merge(tb_transport, on=["country", "year"], how="outer") + + # Improve format. + tb = tb.format() + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, yaml_params={"dollar_year": DOLLAR_YEAR} + ) + + # Save changes in the new garden dataset. + ds_garden.save() diff --git a/etl/steps/data/garden/ihme_gbd/2023-06-14/prevalence_dalys_world.py b/etl/steps/data/garden/ihme_gbd/2023-06-14/prevalence_dalys_world.py index 5c3ec2b8b5d..0df36d230f3 100644 --- a/etl/steps/data/garden/ihme_gbd/2023-06-14/prevalence_dalys_world.py +++ b/etl/steps/data/garden/ihme_gbd/2023-06-14/prevalence_dalys_world.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + from typing import cast import owid.catalog.processing as pr diff --git a/etl/steps/data/garden/ihme_gbd/2024-05-20/gbd_child_mortality.py b/etl/steps/data/garden/ihme_gbd/2024-05-20/gbd_child_mortality.py index 4b44b54ad39..5d906e2d871 100644 --- a/etl/steps/data/garden/ihme_gbd/2024-05-20/gbd_child_mortality.py +++ b/etl/steps/data/garden/ihme_gbd/2024-05-20/gbd_child_mortality.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - from owid.catalog import Table from owid.catalog import processing as pr diff --git a/etl/steps/data/garden/irena/2023-06-26/renewable_electricity_capacity.py b/etl/steps/data/garden/irena/2023-06-26/renewable_electricity_capacity.py index c5d90da8de3..ed2c4dfacb9 100644 --- a/etl/steps/data/garden/irena/2023-06-26/renewable_electricity_capacity.py +++ b/etl/steps/data/garden/irena/2023-06-26/renewable_electricity_capacity.py @@ -1,6 +1,4 @@ -"""Create a dataset of renewable electricity capacity using IRENA's Renewable Electricity Capacity and Generation. - -""" +"""Create a dataset of renewable electricity capacity using IRENA's Renewable Electricity Capacity and Generation.""" from typing import List diff --git a/etl/steps/data/garden/irena/2023-12-12/renewable_electricity_capacity.py b/etl/steps/data/garden/irena/2023-12-12/renewable_electricity_capacity.py index eb8997276af..e45814b4d28 100644 --- a/etl/steps/data/garden/irena/2023-12-12/renewable_electricity_capacity.py +++ b/etl/steps/data/garden/irena/2023-12-12/renewable_electricity_capacity.py @@ -1,6 +1,5 @@ -"""Create a dataset of renewable electricity capacity using IRENA's Renewable Electricity Capacity and Generation. +"""Create a dataset of renewable electricity capacity using IRENA's Renewable Electricity Capacity and Generation.""" -""" from etl.data_helpers import geo from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/garden/irena/2024-11-01/renewable_capacity_statistics.py b/etl/steps/data/garden/irena/2024-11-01/renewable_capacity_statistics.py index 6884b7c3eb7..6ba15bb6246 100644 --- a/etl/steps/data/garden/irena/2024-11-01/renewable_capacity_statistics.py +++ b/etl/steps/data/garden/irena/2024-11-01/renewable_capacity_statistics.py @@ -66,6 +66,7 @@ Yes|On-grid |Wind |Onshore wind |Onshore wind energy -> On-grid |Onshore wind """ + import owid.catalog.processing as pr from owid.catalog import Table from owid.datautils.dataframes import map_series diff --git a/etl/steps/data/garden/lgbt_rights/2024-06-03/equaldex.py b/etl/steps/data/garden/lgbt_rights/2024-06-03/equaldex.py index 9a18be819ec..2d593289bf6 100644 --- a/etl/steps/data/garden/lgbt_rights/2024-06-03/equaldex.py +++ b/etl/steps/data/garden/lgbt_rights/2024-06-03/equaldex.py @@ -181,9 +181,9 @@ def run(dest_dir: str) -> None: tb = geo.harmonize_countries(df=tb, countries_file=paths.country_mapping_path) # NOTE: Correcting data for France in changing_gender to "Legal, surgery required" between 1992 and 2018 - tb.loc[ - (tb["country"] == "France") & (tb["year"] >= 1992) & (tb["year"] < 2018), "changing_gender" - ] = "Legal, surgery required" + tb.loc[(tb["country"] == "France") & (tb["year"] >= 1992) & (tb["year"] < 2018), "changing_gender"] = ( + "Legal, surgery required" + ) # Select only sovereign countries tb = select_only_sovereign_countries(tb=tb, tb_sovereign_countries=tb_sovereign_countries) diff --git a/etl/steps/data/garden/lgbt_rights/2024-06-11/criminalization_mignot.py b/etl/steps/data/garden/lgbt_rights/2024-06-11/criminalization_mignot.py index 283849a888f..964b5290ae8 100644 --- a/etl/steps/data/garden/lgbt_rights/2024-06-11/criminalization_mignot.py +++ b/etl/steps/data/garden/lgbt_rights/2024-06-11/criminalization_mignot.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - from typing import List import owid.catalog.processing as pr diff --git a/etl/steps/data/garden/malnutrition/2022-10-18/malnutrition.py b/etl/steps/data/garden/malnutrition/2022-10-18/malnutrition.py index 16a3240423c..0038b585bfc 100644 --- a/etl/steps/data/garden/malnutrition/2022-10-18/malnutrition.py +++ b/etl/steps/data/garden/malnutrition/2022-10-18/malnutrition.py @@ -1,5 +1,5 @@ -"""Combine WDI's prevalence of stunting, wasting and underweight in children under five. Convert this into absolute numbers using UNWPP population data. -""" +"""Combine WDI's prevalence of stunting, wasting and underweight in children under five. Convert this into absolute numbers using UNWPP population data.""" + from owid import catalog from shared import CURRENT_DIR diff --git a/etl/steps/data/garden/migration/2024-11-19/migration_distance.countries.json b/etl/steps/data/garden/migration/2024-11-19/migration_distance.countries.json new file mode 100644 index 00000000000..e3cbe39c0a3 --- /dev/null +++ b/etl/steps/data/garden/migration/2024-11-19/migration_distance.countries.json @@ -0,0 +1,260 @@ +{ + "Afghanistan": "Afghanistan", + "Albania": "Albania", + "Algeria": "Algeria", + "Angola": "Angola", + "Antarctica": "Antarctica", + "Argentina": "Argentina", + "Armenia": "Armenia", + "Australia": "Australia", + "Austria": "Austria", + "Azerbaijan": "Azerbaijan", + "Bahamas": "Bahamas", + "Bangladesh": "Bangladesh", + "Belarus": "Belarus", + "Belgium": "Belgium", + "Belize": "Belize", + "Benin": "Benin", + "Bhutan": "Bhutan", + "Bolivia": "Bolivia", + "Botswana": "Botswana", + "Brazil": "Brazil", + "Brunei": "Brunei", + "Bulgaria": "Bulgaria", + "Burkina Faso": "Burkina Faso", + "Burundi": "Burundi", + "Cambodia": "Cambodia", + "Cameroon": "Cameroon", + "Canada": "Canada", + "Chad": "Chad", + "Chile": "Chile", + "China": "China", + "Colombia": "Colombia", + "Congo": "Congo", + "Costa Rica": "Costa Rica", + "Croatia": "Croatia", + "Cuba": "Cuba", + "Cyprus": "Cyprus", + "Czechia": "Czechia", + "Denmark": "Denmark", + "Djibouti": "Djibouti", + "Ecuador": "Ecuador", + "Egypt": "Egypt", + "El Salvador": "El Salvador", + "Eritrea": "Eritrea", + "Estonia": "Estonia", + "Ethiopia": "Ethiopia", + "Fiji": "Fiji", + "Finland": "Finland", + "France": "France", + "Gabon": "Gabon", + "Gambia": "Gambia", + "Georgia": "Georgia", + "Germany": "Germany", + "Ghana": "Ghana", + "Greece": "Greece", + "Greenland": "Greenland", + "Guatemala": "Guatemala", + "Guinea": "Guinea", + "Guinea-Bissau": "Guinea-Bissau", + "Guyana": "Guyana", + "Haiti": "Haiti", + "Honduras": "Honduras", + "Hungary": "Hungary", + "Iceland": "Iceland", + "India": "India", + "Indonesia": "Indonesia", + "Iran": "Iran", + "Iraq": "Iraq", + "Ireland": "Ireland", + "Israel": "Israel", + "Italy": "Italy", + "Jamaica": "Jamaica", + "Japan": "Japan", + "Jordan": "Jordan", + "Kazakhstan": "Kazakhstan", + "Kenya": "Kenya", + "Kosovo": "Kosovo", + "Kuwait": "Kuwait", + "Kyrgyzstan": "Kyrgyzstan", + "Laos": "Laos", + "Latvia": "Latvia", + "Lebanon": "Lebanon", + "Lesotho": "Lesotho", + "Liberia": "Liberia", + "Libya": "Libya", + "Lithuania": "Lithuania", + "Luxembourg": "Luxembourg", + "Madagascar": "Madagascar", + "Malawi": "Malawi", + "Malaysia": "Malaysia", + "Mali": "Mali", + "Mauritania": "Mauritania", + "Mexico": "Mexico", + "Moldova": "Moldova", + "Mongolia": "Mongolia", + "Montenegro": "Montenegro", + "Morocco": "Morocco", + "Mozambique": "Mozambique", + "Myanmar": "Myanmar", + "Namibia": "Namibia", + "Nepal": "Nepal", + "Netherlands": "Netherlands", + "New Caledonia": "New Caledonia", + "New Zealand": "New Zealand", + "Nicaragua": "Nicaragua", + "Niger": "Niger", + "Nigeria": "Nigeria", + "North Korea": "North Korea", + "North Macedonia": "North Macedonia", + "Norway": "Norway", + "Oman": "Oman", + "Pakistan": "Pakistan", + "Palestine": "Palestine", + "Panama": "Panama", + "Papua New Guinea": "Papua New Guinea", + "Paraguay": "Paraguay", + "Peru": "Peru", + "Philippines": "Philippines", + "Poland": "Poland", + "Portugal": "Portugal", + "Puerto Rico": "Puerto Rico", + "Qatar": "Qatar", + "Romania": "Romania", + "Russia": "Russia", + "Rwanda": "Rwanda", + "Saudi Arabia": "Saudi Arabia", + "Senegal": "Senegal", + "Serbia": "Serbia", + "Sierra Leone": "Sierra Leone", + "Slovakia": "Slovakia", + "Slovenia": "Slovenia", + "Somalia": "Somalia", + "Somaliland": "Somaliland", + "South Africa": "South Africa", + "South Korea": "South Korea", + "Spain": "Spain", + "Sri Lanka": "Sri Lanka", + "Sudan": "Sudan", + "Suriname": "Suriname", + "Sweden": "Sweden", + "Switzerland": "Switzerland", + "Syria": "Syria", + "Taiwan": "Taiwan", + "Tajikistan": "Tajikistan", + "Tanzania": "Tanzania", + "Thailand": "Thailand", + "Togo": "Togo", + "Trinidad and Tobago": "Trinidad and Tobago", + "Tunisia": "Tunisia", + "Turkey": "Turkey", + "Turkmenistan": "Turkmenistan", + "Uganda": "Uganda", + "Ukraine": "Ukraine", + "United Arab Emirates": "United Arab Emirates", + "United Kingdom": "United Kingdom", + "Uruguay": "Uruguay", + "Uzbekistan": "Uzbekistan", + "Vanuatu": "Vanuatu", + "Venezuela": "Venezuela", + "Vietnam": "Vietnam", + "Yemen": "Yemen", + "Zambia": "Zambia", + "Zimbabwe": "Zimbabwe", + "Fr. S. Antarctic Lands": "French Southern Territories", + "American Samoa": "American Samoa", + "Andorra": "Andorra", + "Anguilla": "Anguilla", + "Antigua and Barbuda": "Antigua and Barbuda", + "Aruba": "Aruba", + "Bahrain": "Bahrain", + "Barbados": "Barbados", + "Bermuda": "Bermuda", + "British Indian Ocean Territory": "British Indian Ocean Territory", + "British Virgin Islands": "British Virgin Islands", + "Cayman Islands": "Cayman Islands", + "Comoros": "Comoros", + "Cook Islands": "Cook Islands", + "Dominica": "Dominica", + "Faroe Islands": "Faroe Islands", + "French Polynesia": "French Polynesia", + "Gibraltar": "Gibraltar", + "Grenada": "Grenada", + "Guam": "Guam", + "Guernsey": "Guernsey", + "Isle of Man": "Isle of Man", + "Jersey": "Jersey", + "Kiribati": "Kiribati", + "Liechtenstein": "Liechtenstein", + "Maldives": "Maldives", + "Malta": "Malta", + "Marshall Islands": "Marshall Islands", + "Mauritius": "Mauritius", + "Monaco": "Monaco", + "Nauru": "Nauru", + "Niue": "Niue", + "Northern Mariana Islands": "Northern Mariana Islands", + "Palau": "Palau", + "Saint Barthelemy": "Saint Barthelemy", + "Saint Helena": "Saint Helena", + "Saint Kitts and Nevis": "Saint Kitts and Nevis", + "Saint Lucia": "Saint Lucia", + "Saint Pierre and Miquelon": "Saint Pierre and Miquelon", + "Saint Vincent and the Grenadines": "Saint Vincent and the Grenadines", + "Samoa": "Samoa", + "San Marino": "San Marino", + "Sao Tome and Principe": "Sao Tome and Principe", + "Seychelles": "Seychelles", + "Singapore": "Singapore", + "Tonga": "Tonga", + "Turks and Caicos Islands": "Turks and Caicos Islands", + "Tuvalu": "Tuvalu", + "United States Virgin Islands": "United States Virgin Islands", + "Vatican": "Vatican", + "Wallis and Futuna": "Wallis and Futuna", + "Aland Islands": "Aland Islands", + "Bosnia and Herzegovina": "Bosnia and Herzegovina", + "Cape Verde": "Cape Verde", + "Central African Republic": "Central African Republic", + "Cote d'Ivoire": "Cote d'Ivoire", + "Curacao": "Curacao", + "Democratic Republic of Congo": "Democratic Republic of Congo", + "Dominican Republic": "Dominican Republic", + "East Timor": "East Timor", + "Equatorial Guinea": "Equatorial Guinea", + "Eswatini": "Eswatini", + "Falkland Islands": "Falkland Islands", + "Heard Island and McDonald Islands": "Heard Island and McDonald Islands", + "Hong Kong": "Hong Kong", + "Macao": "Macao", + "Micronesia (country)": "Micronesia (country)", + "Montserrat": "Montserrat", + "Norfolk Island": "Norfolk Island", + "Northern Cyprus": "Northern Cyprus", + "Pitcairn": "Pitcairn", + "Saint Martin (French part)": "Saint Martin (French part)", + "Sint Maarten (Dutch part)": "Sint Maarten (Dutch part)", + "Solomon Islands": "Solomon Islands", + "South Sudan": "South Sudan", + "United States": "United States", + "Western Sahara": "Western Sahara", + "USNB Guantanamo Bay": "Guantanamo Bay", + "Akrotiri": "Akrotiri", + "Serranilla Bank": "Serranilla Bank", + "S. Geo. and the Is.": "South Georgia and the South Sandwich Islands", + "Bir Tawil": "Bir Tawil", + "Scarborough Reef": "Scarborough Reef", + "Spratly Is.": "Spratly Islands", + "Coral Sea Is.": "Coral Sea Islands", + "U.S. Minor Outlying Is.": "U.S. Minor Outlying Islands", + "Siachen Glacier": "Siachen Glacier", + "Indian Ocean Ter.": "Indian Ocean Territories", + "Southern Patagonian Ice Field": "Southern Patagonian Ice Field", + "Cyprus U.N. Buffer Zone": "Cyprus U.N. Buffer Zone", + "Dhekelia": "Dhekelia", + "Clipperton I.": "Clipperton Island", + "Ashmore and Cartier Is.": "Ashmore and Cartier Islands", + "Bajo Nuevo Bank": "Bajo Nuevo Bank", + "Baikonur": "Baikonur", + "Brazilian I.": "Brazilian Island" +} diff --git a/etl/steps/data/garden/migration/2024-11-19/migration_distance.meta.yml b/etl/steps/data/garden/migration/2024-11-19/migration_distance.meta.yml new file mode 100644 index 00000000000..23950136cf1 --- /dev/null +++ b/etl/steps/data/garden/migration/2024-11-19/migration_distance.meta.yml @@ -0,0 +1,37 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Migration + processing_level: major + description_processing: >- + - The migration distance indicator measures the median distance between the borders of the origin and destination country of migrants. It is calculated by first taking the minimum distance between the borders of origin and destination countries for each migrant, and then taking the median of these minimum distances. + - The indicator is based on the UN DESA International Migrant Stock data, which provides the number of international migrants by country of origin and destination. The geospatial data for the borders of countries is retrieved from the Natural Earth project (https://www.naturalearthdata.com/downloads/110m-cultural-vectors/), specifically the "Admin 0 - countries and tiny country points" datasets. + + +dataset: + update_period_days: 365 + title: Median migration distance (between borders) + description: |- + This data set includes the median distance between the borders of the origin and destination country of migrants. It is calculated by first taking the minimum distance between the borders of origin and destination countries for each migrant, and then calculating the median of these minimum distances. + + The number of international migrants and their origin and destination countries are based on the UN DESA International Migrant Stock data. The geospatial data for the borders of countries is retrieved from the Natural Earth project (https://www.naturalearthdata.com/downloads/110m-cultural-vectors/), specifically the admin 0 - countries and tiny country points datasets. + + + +tables: + migration_distance: + variables: + median_distance: + title: Median migration distance for emigrants leaving this country + description_short: |- + The median distance between the borders of the origin and destination country of migrants. It is calculated by first taking the minimum distance between the borders of origin and destination countries for each migrant, and then calculating the median of these minimum distances. + unit: kilometers + short_unit: km + total_emigrants: + title: Total number of migrants leaving this country + description_short: The total number of international migrants leaving this country. + unit: persons + short_unit: "" + diff --git a/etl/steps/data/garden/migration/2024-11-19/migration_distance.py b/etl/steps/data/garden/migration/2024-11-19/migration_distance.py new file mode 100644 index 00000000000..f48fd00d872 --- /dev/null +++ b/etl/steps/data/garden/migration/2024-11-19/migration_distance.py @@ -0,0 +1,139 @@ +"""Load a meadow dataset and create a garden dataset.""" + +import warnings + +import geopandas as gpd +import pandas as pd +import structlog +from geopy.distance import geodesic +from shapely import wkt +from shapely.ops import nearest_points +from tqdm import tqdm + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +LOG = structlog.get_logger() + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("migrant_stock") + ds_nat_earth = paths.load_dataset("nat_earth_110") + + # Read table from meadow dataset. + tb = ds_meadow["migrant_stock_dest_origin"].reset_index() + tb_countries = ds_nat_earth["nat_earth_110"].reset_index() + + # Read natural earth data + # Convert the geometry string column to a Shapely object. + tb_countries["geometry"] = tb_countries["geometry"].apply(wkt.loads) + world = gpd.GeoDataFrame(tb_countries, geometry="geometry") + + # use World Geodetic System 1984 as projection + world = world.set_crs("EPSG:4326") + + # harmonize country names + world = geo.harmonize_countries( + df=world, # type: ignore + country_col="name", + countries_file=paths.country_mapping_path, + ) + + # Calculate distance matrix (in km) (catch warnings to ignore "invalid value encountered" warning) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=RuntimeWarning) + distance_matrix = calculate_distance_matrix(world) + + ## Add distances to migration flows table + # Remove countries not included in nat earth data and "Other" from country destination or country origin columns + cty_no_data = [ + "Other", + "Tokelau", + "Bonaire Sint Eustatius and Saba", + "French Guiana", + "Guadeloupe", + "Martinique", + "Reunion", + "Channel Islands", + "Mayotte", + ] + + cty_data = [cty for cty in tb["country_origin"].unique() if cty not in cty_no_data] + tb = tb[(tb["country_destination"].isin(cty_data)) & (tb["country_origin"].isin(cty_data))] + + # Add distance to the table + tb["distance"] = tb.apply( + lambda row: distance_matrix.loc[row["country_origin"], row["country_destination"]], axis=1 + ) + tb["distance"] = tb["distance"].apply(get_min_distance).astype("Float64") + + migrant_groups = tb.groupby(["country_origin", "year"]) + med_distance = migrant_groups.apply(calc_median).reset_index() + med_distance["median_distance"] = med_distance[0].apply(lambda x: x[0]) + med_distance["total_emigrants"] = med_distance[0].apply(lambda x: x[1]) + med_distance = med_distance.drop(columns=[0]).copy_metadata(tb) + + med_distance.metadata.dataset.short_name = "migration_distance" + med_distance.metadata.short_name = "migration_distance" + + for col in med_distance.columns: + med_distance[col].metadata.origins = tb["country_origin"].m.origins + tb_countries["name"].m.origins + + med_distance = med_distance.format(["country_origin", "year"]) + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset(dest_dir, tables=[med_distance], check_variables_metadata=True, default_metadata=None) + + # Save changes in the new garden dataset. + ds_garden.save() + + +def get_min_distance(distance): + """Get the minimum distance between two countries. + Sometimes the distance is a DataFrame or Series (if country is represented by multiple polygons). Then we take the minimum distance.""" + if isinstance(distance, pd.Series): + return distance.min() + elif isinstance(distance, pd.DataFrame): + return distance.min().min() + else: + return distance + + +def calc_median(group, col="distance"): + """Calculate the median distance for each country origin and year.""" + group = group.sort_values(by=col) + group["cumulative_journeys"] = group["migrants_all_sexes"].cumsum() + total_journeys = group["migrants_all_sexes"].sum() + median_journey = total_journeys / 2 + median_dist = group[group["cumulative_journeys"] >= median_journey].iloc[0]["distance"] + return median_dist, total_journeys + + +def calculate_distance_matrix(world): + # Create an empty distance matrix + distance_matrix = pd.DataFrame(index=world["name"], columns=world["name"]) + + for i, row1 in tqdm(world.iterrows(), total=len(world), desc="Calculating distance matrix"): + for j, row2 in world.iterrows(): + if i == j: + distance_matrix.iloc[i, j] = 0 # Distance to itself + elif i > j: + distance_matrix.iloc[i, j] = distance_matrix.iloc[j, i] # Distance is symmetric + else: + # Get the nearest points between two geometries + point1, point2 = nearest_points(row1.geometry, row2.geometry) # type: ignore + + # Calculate geodesic distance between the nearest points + distance_matrix.iloc[i, j] = geodesic((point1.y, point1.x), (point2.y, point2.x)).kilometers # type: ignore + + return distance_matrix diff --git a/etl/steps/data/garden/minerals/2024-07-15/minerals.meta.yml b/etl/steps/data/garden/minerals/2024-07-15/minerals.meta.yml index 21a8f682331..51a5d64a43b 100644 --- a/etl/steps/data/garden/minerals/2024-07-15/minerals.meta.yml +++ b/etl/steps/data/garden/minerals/2024-07-15/minerals.meta.yml @@ -999,6 +999,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of aluminum, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Aluminum + shortUnit: $ presentation: title_public: Aluminum unit value unit_value_antimony_mine_constant_1998_usd_per_tonne: @@ -1007,6 +1010,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [antimony](#dod:antimony), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Antimony + shortUnit: $ presentation: title_public: Antimony unit value unit_value_arsenic_processing_constant_1998_usd_per_tonne: @@ -1015,6 +1021,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of arsenic, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Arsenic + shortUnit: $ presentation: title_public: Arsenic unit value unit_value_asbestos_mine_constant_1998_usd_per_tonne: @@ -1023,6 +1032,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of asbestos, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Asbestos + shortUnit: $ presentation: title_public: Asbestos unit value unit_value_barite_mine_constant_1998_usd_per_tonne: @@ -1031,6 +1043,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [barite](#dod:barite), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Barite + shortUnit: $ presentation: title_public: Barite unit value unit_value_bauxite_mine_constant_1998_usd_per_tonne: @@ -1039,6 +1054,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [bauxite](#dod:bauxite), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Bauxite + shortUnit: $ presentation: title_public: Bauxite unit value unit_value_beryllium_mine_constant_1998_usd_per_tonne: @@ -1047,6 +1065,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [beryllium](#dod:beryllium), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Beryllium + shortUnit: $ presentation: title_public: Beryllium unit value unit_value_bismuth_mine_constant_1998_usd_per_tonne: @@ -1055,6 +1076,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [bismuth](#dod:bismuth), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Bismuth + shortUnit: $ presentation: title_public: Bismuth unit value unit_value_boron_mine_constant_1998_usd_per_tonne: @@ -1063,6 +1087,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [boron](#dod:boron), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Boron + shortUnit: $ presentation: title_public: Boron unit value unit_value_cadmium_refinery_constant_1998_usd_per_tonne: @@ -1071,6 +1098,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [cadmium](#dod:cadmium), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Cadmium + shortUnit: $ presentation: title_public: Refined cadmium unit value unit_value_cement_processing_constant_1998_usd_per_tonne: @@ -1079,6 +1109,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of cement, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Cement + shortUnit: $ presentation: title_public: Cement unit value # unit_value_chromium_mine_constant_1998_usd_per_tonne: @@ -1087,6 +1120,9 @@ tables: # short_unit: $/t # description_short: Value of 1 tonne of chromium, in constant 1998 US$ per tonne. # description_key: *unit-value-description-key + # display: + # name: Chromium + # shortUnit: $ # presentation: # title_public: Chromium unit value unit_value_cobalt_value_constant_1998_usd_per_tonne: @@ -1095,6 +1131,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [cobalt](#dod:cobalt), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Cobalt + shortUnit: $ presentation: title_public: Cobalt unit value unit_value_copper_mine_constant_1998_usd_per_tonne: @@ -1103,6 +1142,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of copper, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Copper + shortUnit: $ presentation: title_public: Copper unit value unit_value_diamond_mine_and_synthetic__industrial_constant_1998_usd_per_tonne: @@ -1111,6 +1153,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of diamond, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Diamond + shortUnit: $ presentation: title_public: Diamond unit value unit_value_feldspar_mine_constant_1998_usd_per_tonne: @@ -1119,6 +1164,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [feldspar](#dod:feldspar), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Feldspar + shortUnit: $ presentation: title_public: Feldspar unit value unit_value_fluorspar_mine_constant_1998_usd_per_tonne: @@ -1127,6 +1175,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [fluorspar](#dod:fluorspar), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Fluorspar + shortUnit: $ presentation: title_public: Fluorspar unit value unit_value_gallium_refinery_constant_1998_usd_per_tonne: @@ -1135,6 +1186,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [gallium](#dod:gallium), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Gallium + shortUnit: $ presentation: title_public: Refined gallium unit value unit_value_garnet_mine_constant_1998_usd_per_tonne: @@ -1143,6 +1197,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [garnet](#dod:garnet), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Garnet + shortUnit: $ presentation: title_public: Garnet unit value unit_value_germanium_refinery_constant_1998_usd_per_tonne: @@ -1151,6 +1208,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [germanium](#dod:germanium), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Germanium + shortUnit: $ presentation: title_public: Refined germanium unit value unit_value_gold_mine_constant_1998_usd_per_tonne: @@ -1159,6 +1219,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of gold, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Gold + shortUnit: $ presentation: title_public: Gold unit value unit_value_graphite_mine_constant_1998_usd_per_tonne: @@ -1167,6 +1230,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [graphite](#dod:graphite), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Graphite + shortUnit: $ presentation: title_public: Graphite unit value unit_value_gypsum_mine_constant_1998_usd_per_tonne: @@ -1175,6 +1241,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [gypsum](#dod:gypsum), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Gypsum + shortUnit: $ presentation: title_public: Gypsum unit value unit_value_helium_mine_constant_1998_usd_per_tonne: @@ -1183,6 +1252,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of helium, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Helium + shortUnit: $ presentation: title_public: Helium unit value unit_value_iron_ore_mine__crude_ore_constant_1998_usd_per_tonne: @@ -1191,6 +1263,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of iron ore, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Iron ore + shortUnit: $ presentation: title_public: Iron ore (crude ore) unit value unit_value_iron_smelter__pig_iron_constant_1998_usd_per_tonne: @@ -1199,6 +1274,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of iron, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Iron + shortUnit: $ presentation: title_public: Iron unit value unit_value_lead_mine_constant_1998_usd_per_tonne: @@ -1207,6 +1285,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of lead, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Lead + shortUnit: $ presentation: title_public: Lead unit value unit_value_lime_processing_constant_1998_usd_per_tonne: @@ -1215,14 +1296,31 @@ tables: short_unit: $/t description_short: Value of 1 tonne of lime, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Lime + shortUnit: $ presentation: title_public: Lime unit value + unit_value_lithium_mine_constant_1998_usd_per_tonne: + title: unit value|Lithium|Mine|constant 1998 US$ per tonne + unit: constant 1998 US$ per tonne + short_unit: $/t + description_short: Value of 1 tonne of lithium, in constant 1998 US$ per tonne. + description_key: *unit-value-description-key + display: + name: Lithium + shortUnit: $ + presentation: + title_public: Lithium unit value unit_value_magnesium_compounds_mine_constant_1998_usd_per_tonne: title: unit value|Magnesium compounds|Mine|constant 1998 US$ per tonne unit: constant 1998 US$ per tonne short_unit: $/t description_short: Value of 1 tonne of magnesium compounds, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Magnesium compounds + shortUnit: $ presentation: title_public: Magnesium compounds unit value unit_value_magnesium_metal_smelter_constant_1998_usd_per_tonne: @@ -1231,6 +1329,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of magnesium metal, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Magnesium metal + shortUnit: $ presentation: title_public: Magnesium metal unit value unit_value_manganese_mine_constant_1998_usd_per_tonne: @@ -1239,6 +1340,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of manganese, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Manganese + shortUnit: $ presentation: title_public: Manganese unit value unit_value_mercury_mine_constant_1998_usd_per_tonne: @@ -1247,6 +1351,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of mercury, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Mercury + shortUnit: $ presentation: title_public: Mercury unit value unit_value_mica_mine__scrap_and_flake_constant_1998_usd_per_tonne: @@ -1255,6 +1362,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [mica](#dod:mica) (scrap and flake), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Mica (scrap and flake) + shortUnit: $ presentation: title_public: Mica (scrap and flake) unit value unit_value_mica_mine__sheet_constant_1998_usd_per_tonne: @@ -1263,6 +1373,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [mica](#dod:mica) (sheet), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Mica (sheet) + shortUnit: $ presentation: title_public: Mica (sheet) unit value unit_value_molybdenum_mine_constant_1998_usd_per_tonne: @@ -1271,6 +1384,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [molybdenum](#dod:molybdenum), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Molybdenum + shortUnit: $ presentation: title_public: Molybdenum unit value unit_value_nickel_mine_constant_1998_usd_per_tonne: @@ -1279,6 +1395,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of nickel, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Nickel + shortUnit: $ presentation: title_public: Nickel unit value unit_value_niobium_mine_constant_1998_usd_per_tonne: @@ -1287,6 +1406,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [niobium](#dod:niobium), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Niobium + shortUnit: $ presentation: title_public: Niobium unit value unit_value_nitrogen_fixed_ammonia_constant_1998_usd_per_tonne: @@ -1295,6 +1417,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of nitrogen, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Nitrogen + shortUnit: $ presentation: title_public: Nitrogen unit value unit_value_phosphate_rock_mine_constant_1998_usd_per_tonne: @@ -1303,14 +1428,31 @@ tables: short_unit: $/t description_short: Value of 1 tonne of phosphate rock, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Phosphate rock + shortUnit: $ presentation: title_public: Phosphate rock unit value + unit_value_rare_earths_mine_constant_1998_usd_per_tonne: + title: unit value|Rare earths|Mine|constant 1998 US$ per tonne + unit: constant 1998 US$ per tonne + short_unit: $/t + description_short: Value of 1 tonne of rare earths, in constant 1998 US$ per tonne. + description_key: *unit-value-description-key + display: + name: Rare earths + shortUnit: $ + presentation: + title_public: Rare earths unit value unit_value_salt_mine_constant_1998_usd_per_tonne: title: unit value|Salt|Mine|constant 1998 US$ per tonne unit: constant 1998 US$ per tonne short_unit: $/t description_short: Value of 1 tonne of salt, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Salt + shortUnit: $ presentation: title_public: Salt unit value unit_value_sand_and_gravel_mine__construction_constant_1998_usd_per_tonne: @@ -1319,6 +1461,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of sand and gravel, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Sand and gravel + shortUnit: $ presentation: title_public: Sand and gravel (construction) unit value unit_value_sand_and_gravel_mine__industrial_constant_1998_usd_per_tonne: @@ -1327,6 +1472,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of sand and gravel, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Sand and gravel + shortUnit: $ presentation: title_public: Sand and gravel (industrial) unit value unit_value_selenium_refinery_constant_1998_usd_per_tonne: @@ -1335,6 +1483,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of selenium, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Selenium + shortUnit: $ presentation: title_public: Refined selenium unit value unit_value_silicon_processing_constant_1998_usd_per_tonne: @@ -1343,6 +1494,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of silicon, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Silicon + shortUnit: $ presentation: title_public: Silicon unit value unit_value_silver_mine_constant_1998_usd_per_tonne: @@ -1351,6 +1505,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of silver, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Silver + shortUnit: $ presentation: title_public: Silver unit value unit_value_steel_processing__crude_constant_1998_usd_per_tonne: @@ -1359,6 +1516,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of steel, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Steel + shortUnit: $ presentation: title_public: Steel unit value unit_value_strontium_mine_constant_1998_usd_per_tonne: @@ -1367,6 +1527,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [strontium](#dod:strontium), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Strontium + shortUnit: $ presentation: title_public: Strontium unit value unit_value_sulfur_processing_constant_1998_usd_per_tonne: @@ -1375,6 +1538,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of sulfur, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Sulfur + shortUnit: $ presentation: title_public: Sulfur unit value unit_value_talc_and_pyrophyllite_mine_constant_1998_usd_per_tonne: @@ -1383,6 +1549,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of talc and pyrophyllite, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Talc and pyrophyllite + shortUnit: $ presentation: title_public: Talc and pyrophyllite unit value unit_value_tantalum_mine_constant_1998_usd_per_tonne: @@ -1391,6 +1560,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [tantalum](#dod:tantalum), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Tantalum + shortUnit: $ presentation: title_public: Tantalum unit value unit_value_tin_mine_constant_1998_usd_per_tonne: @@ -1399,6 +1571,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of tin, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Tin + shortUnit: $ presentation: title_public: Tin unit value unit_value_tungsten_mine_constant_1998_usd_per_tonne: @@ -1407,6 +1582,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [tungsten](#dod:tungsten), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Tungsten + shortUnit: $ presentation: title_public: Tungsten unit value unit_value_vanadium_mine_constant_1998_usd_per_tonne: @@ -1415,6 +1593,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of [vanadium](#dod:vanadium), in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Vanadium + shortUnit: $ presentation: title_public: Vanadium unit value unit_value_zinc_mine_constant_1998_usd_per_tonne: @@ -1423,6 +1604,9 @@ tables: short_unit: $/t description_short: Value of 1 tonne of zinc, in constant 1998 US$ per tonne. description_key: *unit-value-description-key + display: + name: Zinc + shortUnit: $ presentation: title_public: Zinc unit value share_of_global_production_aluminum_smelter_tonnes: diff --git a/etl/steps/data/garden/minerals/2024-07-15/minerals.py b/etl/steps/data/garden/minerals/2024-07-15/minerals.py index 18cd9548a90..9b17a5b2274 100644 --- a/etl/steps/data/garden/minerals/2024-07-15/minerals.py +++ b/etl/steps/data/garden/minerals/2024-07-15/minerals.py @@ -17,6 +17,7 @@ NOTE: We allow this where one or a few BGS data points are significantly larger than USGS (e.g. Lead). But we do not do this if many points in BGS data are consistently larger than USGS (e.g. Graphite). """ + import warnings from typing import List, Optional, Tuple @@ -237,9 +238,7 @@ ] # Columns to plot with the individual data sources differentiated. PLOT_TO_COMPARE_DATA_SOURCES = [ - # "production|Chromium|Mine|tonnes", - # 'production|Titanium|Mine, ilmenite|tonnes', - # 'production|Helium|Mine|tonnes', + # "production|Rhenium|Mine|tonnes", ] @@ -591,9 +590,9 @@ def combine_data( tb.loc[ (tb["country"] != "World") & (tb["year"].isin([1997, 2006, 2008, 2010, 2011])), "production|Barite|Mine|tonnes" ] = None - tb.loc[ - (tb["country"] != "World") & (tb["year"].isin([1972, 1973, 1975])), "production|Fluorspar|Mine|tonnes" - ] = None + tb.loc[(tb["country"] != "World") & (tb["year"].isin([1972, 1973, 1975])), "production|Fluorspar|Mine|tonnes"] = ( + None + ) tb.loc[(tb["country"] != "World") & (tb["year"] < 1975), "production|Gold|Mine|tonnes"] = None tb.loc[ (tb["country"] != "World") & (tb["year"].isin([2014])), @@ -607,6 +606,10 @@ def combine_data( (tb["country"] != "World") & (tb["year"].isin([1992])), "production|Cobalt|Refinery|tonnes", ] = None + tb.loc[ + (tb["country"] != "World") & (tb["year"].isin([1977, 1978, 1979, 1983])), + "production|Iodine|Mine|tonnes", + ] = None #################################################################################################################### diff --git a/etl/steps/data/garden/nasa/2024-05-31/ozone_hole_area.py b/etl/steps/data/garden/nasa/2024-05-31/ozone_hole_area.py index 57734005352..4648fea89ae 100644 --- a/etl/steps/data/garden/nasa/2024-05-31/ozone_hole_area.py +++ b/etl/steps/data/garden/nasa/2024-05-31/ozone_hole_area.py @@ -3,7 +3,6 @@ This step adds a new column: "country" with the value "World", since the data in this dataset is only for the world.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/garden/oecd/2023-09-21/plastic_use_projections.py b/etl/steps/data/garden/oecd/2023-09-21/plastic_use_projections.py index a74596a1199..f3d12173054 100644 --- a/etl/steps/data/garden/oecd/2023-09-21/plastic_use_projections.py +++ b/etl/steps/data/garden/oecd/2023-09-21/plastic_use_projections.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + import owid.catalog.processing as pr from etl.data_helpers import geo diff --git a/etl/steps/data/garden/oecd/2023-09-21/plastic_waste_2019.py b/etl/steps/data/garden/oecd/2023-09-21/plastic_waste_2019.py index 71853c2608f..3b0745fdfe0 100644 --- a/etl/steps/data/garden/oecd/2023-09-21/plastic_waste_2019.py +++ b/etl/steps/data/garden/oecd/2023-09-21/plastic_waste_2019.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + import owid.catalog.processing as pr from etl.data_helpers import geo diff --git a/etl/steps/data/garden/oecd/2023-09-21/plastic_waste_2019_by_polymer.py b/etl/steps/data/garden/oecd/2023-09-21/plastic_waste_2019_by_polymer.py index d13772314b8..bb6e2f6a775 100644 --- a/etl/steps/data/garden/oecd/2023-09-21/plastic_waste_2019_by_polymer.py +++ b/etl/steps/data/garden/oecd/2023-09-21/plastic_waste_2019_by_polymer.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + import owid.catalog.processing as pr from etl.data_helpers import geo diff --git a/etl/steps/data/garden/ophi/2024-10-28/multidimensional_poverty_index.meta.yml b/etl/steps/data/garden/ophi/2024-10-28/multidimensional_poverty_index.meta.yml index dc671c36165..bc50164ae0d 100644 --- a/etl/steps/data/garden/ophi/2024-10-28/multidimensional_poverty_index.meta.yml +++ b/etl/steps/data/garden/ophi/2024-10-28/multidimensional_poverty_index.meta.yml @@ -102,8 +102,8 @@ definitions: description_short_intensity: The intensity is the share of indicators in which people in multidimensional poverty are deprived on average. description_short_vulnerable: This is the share of the population that is close to being in multidimensional poverty. description_short_severe: This is the share of the population that is in severe multidimensional poverty. - description_short_censored_headcount_ratio: This is the share of the multidimensionally poor population deprived in the indicator <>. - description_short_uncensored_headcount_ratio: This is the share of the population deprived in the indicator <>. + description_short_censored_headcount_ratio: This is the share of the multidimensionally poor population deprived in the indicator _<>_. + description_short_uncensored_headcount_ratio: This is the share of the population deprived in the indicator _<>_. description_short_area: |- <% if area == "Urban" %> This indicator is calculated for urban areas. @@ -115,9 +115,9 @@ definitions: # For description_key description_key_multidimensional_poverty_complement: |- - grouped into three dimensions of well-being: **health** (using two indicators: nutrition, child mortality), **education** (using two indicators: years of schooling, school attendance), and **living standards** (using five indicators: cooking fuel, sanitation, drinking water, electricity, housing, assets). + grouped into three dimensions of well-being: **health** (using two indicators: nutrition, child mortality), **education** (using two indicators: years of schooling, school attendance), and **living standards** (using six indicators: cooking fuel, sanitation, drinking water, electricity, housing, assets). description_key_multidimensional_poverty: |- - Being _MPI poor_ means that a person lives in a household deprived in a third or more of ten indicators, {definitions.description_key_multidimensional_poverty_complement} + Being in multidimensional poverty means that a person lives in a household deprived in a third or more of ten indicators, {definitions.description_key_multidimensional_poverty_complement} description_key_vulnerable: |- Being _vulnerable_ to multidimensional poverty means that a person lives in a household deprived in 20-33.3% of ten indicators, {definitions.description_key_multidimensional_poverty_complement} description_key_severe: |- @@ -127,14 +127,30 @@ definitions: description_key_weights: |- Each indicator contributes to one of the three dimensions of well-being. Health and education indicators are weighted more (1/6 each) than living standards indicators (1/18 each) so that all three dimensions contribute equally to the overall measure. description_key_mpi: |- - The Multidimensional Poverty Index (MPI) is calculated by multiplying two values: the share of people who are multidimensionally poor and the intensity of their poverty. + The Multidimensional Poverty Index (MPI) is calculated by multiplying two values: the [share of people who are multidimensionally poor](https://ourworldindata.org/grapher/share-multi-poverty) and the [intensity of their poverty](https://ourworldindata.org/grapher/intensity-of-multidimensional-poverty-national). description_key_intensity: |- The intensity of multidimensional poverty is calculated as the average share of indicators in which those counted as MPI poor are deprived. + description_key_flavor_cme: |- + This indicator is a current margin estimate (CME), meaning that it relies on the most recent survey data available for each country. + description_key_flavor_hot: |- + This indicator is a harmonized over time (HOT) estimate. This harmonization seeks to make two or more MPI estimates comparable by aligning the indicator definitions in each survey. description_key_flavor: |- <% if flavor == "Current margin estimate" %> - This indicator is a current margin estimate (CME), based on the most recent survey data available for each country. Look for the harmonized over time (HOT) estimate to see trends over time. + {definitions.description_key_flavor_cme}{definitions.description_key_flavor_link} <% elif flavor == "Harmonized over time" %> - This indicator is a harmonized over time (HOT) estimate. This harmonization seeks to make two or more MPI estimates comparable by aligning the indicator definitions in each survey. Look for the current margin estimate (CME) to see the most recent survey data. + {definitions.description_key_flavor_hot}{definitions.description_key_flavor_link} + <%- endif -%> + description_key_flavor_link: |- + <% if flavor == "Current margin estimate" and area == "National" %> + Look for the [harmonized over time (HOT) estimate](<>) to see trends over time. + <% elif flavor == "Harmonized over time" and area == "National" %> + Look for the [current margin estimate (CME)](<>) to see the most recent survey data. + <%- endif -%> + description_key_flavor_no_links_for_national: |- + <% if flavor == "Current margin estimate" %> + {definitions.description_key_flavor_cme} + <% elif flavor == "Harmonized over time" %> + {definitions.description_key_flavor_hot} <%- endif -%> description_key_indicator_start: |- A person in a household is deprived in the indicator _<>_ if @@ -163,13 +179,28 @@ definitions: description_key_indicator: |- {definitions.description_key_indicator_start} {definitions.description_key_indicator_end} + # For title_variant + title_variant_area: |- + <% if area == "Urban" %> + , Urban area + <% elif area == "Rural" %> + , Rural area + <% elif area == "Camp" %> + , Refugee camps + <%- endif -%> + + title_variant_flavor: |- + <% if flavor == "Current margin estimate" %> + Most recent year + <% elif flavor == "Harmonized over time" %> + Harmonized over time + <%- endif -%> # Learn more about the available fields: # http://docs.owid.io/projects/etl/architecture/metadata/reference/ dataset: update_period_days: 365 - tables: multidimensional_poverty_index: variables: @@ -184,10 +215,10 @@ tables: - "{definitions.description_key_assessments}" - "{definitions.description_key_weights}" - "{definitions.description_key_intensity}" - - "{definitions.description_key_flavor}" + - <% set cme_link = "https://ourworldindata.org/grapher/multidimensional-poverty-index-mpi" %> <% set hot_link = "https://ourworldindata.org/grapher/multidimensional-poverty-index-mpi-hot" %>{definitions.description_key_flavor} presentation: title_public: Multidimensional Poverty Index (MPI) - title_variant: <>, <> + title_variant: "{definitions.title_variant_flavor}{definitions.title_variant_area}" display: name: Multidimensional Poverty Index (MPI) numDecimalPlaces: 3 @@ -202,10 +233,10 @@ tables: - "{definitions.description_key_multidimensional_poverty}" - "{definitions.description_key_assessments}" - "{definitions.description_key_weights}" - - "{definitions.description_key_flavor}" + - <% set cme_link = "https://ourworldindata.org/grapher/share-multi-poverty" %> <% set hot_link = "https://ourworldindata.org/grapher/share-of-population-multidimensionally-poor-hot" %>{definitions.description_key_flavor} presentation: title_public: Share of population living in multidimensional poverty - title_variant: <>, <> + title_variant: "{definitions.title_variant_flavor}{definitions.title_variant_area}" display: name: Share of population living in multidimensional poverty numDecimalPlaces: 1 @@ -221,10 +252,10 @@ tables: - "{definitions.description_key_assessments}" - "{definitions.description_key_intensity}" - "{definitions.description_key_weights}" - - "{definitions.description_key_flavor}" + - <% set cme_link = "https://ourworldindata.org/grapher/intensity-of-multidimensional-poverty" %> <% set hot_link = "https://ourworldindata.org/grapher/intensity-of-multidimensional-poverty-hot" %>{definitions.description_key_flavor} presentation: title_public: Intensity of multidimensional poverty - title_variant: <>, <> + title_variant: "{definitions.title_variant_flavor}{definitions.title_variant_area}" display: name: Intensity of multidimensional poverty numDecimalPlaces: 1 @@ -239,10 +270,10 @@ tables: - "{definitions.description_key_vulnerable}" - "{definitions.description_key_assessments}" - "{definitions.description_key_weights}" - - "{definitions.description_key_flavor}" + - "{definitions.description_key_flavor_no_links_for_national}" presentation: title_public: Share of population vulnerable to multidimensional poverty - title_variant: <>, <> + title_variant: "{definitions.title_variant_flavor}{definitions.title_variant_area}" display: name: Share of population vulnerable to multidimensional poverty numDecimalPlaces: 1 @@ -257,10 +288,10 @@ tables: - "{definitions.description_key_severe}" - "{definitions.description_key_assessments}" - "{definitions.description_key_weights}" - - "{definitions.description_key_flavor}" + - "{definitions.description_key_flavor_no_links_for_national}" presentation: title_public: Share of population living in severe multidimensional poverty - title_variant: <>, <> + title_variant: "{definitions.title_variant_flavor}{definitions.title_variant_area}" display: name: Share of population living in severe multidimensional poverty numDecimalPlaces: 1 @@ -274,10 +305,10 @@ tables: description_key: - "{definitions.description_key_multidimensional_poverty}" - "{definitions.description_key_indicator}" - - "{definitions.description_key_flavor}" + - "{definitions.description_key_flavor_no_links_for_national}" presentation: title_public: Share of population in multidimensional poverty deprived in the indicator <> - title_variant: <>, <> + title_variant: "{definitions.title_variant_flavor}{definitions.title_variant_area}" display: name: Share of population in multidimensional poverty deprived in the indicator <> numDecimalPlaces: 1 @@ -291,12 +322,11 @@ tables: description_key: - "{definitions.description_key_multidimensional_poverty}" - "{definitions.description_key_indicator}" - - "{definitions.description_key_flavor}" + - "{definitions.description_key_flavor_no_links_for_national}" presentation: title_public: Share of population deprived in the indicator <> - title_variant: <>, <> + title_variant: "{definitions.title_variant_flavor}{definitions.title_variant_area}" display: name: Share of population deprived in the indicator <> numDecimalPlaces: 1 <<: *common_display - diff --git a/etl/steps/data/garden/owid/latest/key_indicators/utils.py b/etl/steps/data/garden/owid/latest/key_indicators/utils.py index 90b4c99935f..d265284a930 100644 --- a/etl/steps/data/garden/owid/latest/key_indicators/utils.py +++ b/etl/steps/data/garden/owid/latest/key_indicators/utils.py @@ -1,4 +1,5 @@ """Utils for key_indicators module.""" + import pandas as pd from etl.data_helpers import geo diff --git a/etl/steps/data/garden/papers/2023-07-10/farmer_lafond_2016.py b/etl/steps/data/garden/papers/2023-07-10/farmer_lafond_2016.py index d090e85c25e..95a53109c60 100644 --- a/etl/steps/data/garden/papers/2023-07-10/farmer_lafond_2016.py +++ b/etl/steps/data/garden/papers/2023-07-10/farmer_lafond_2016.py @@ -1,6 +1,4 @@ -"""Harmonize data from Farmer & Lafond (2016) paper on the evolution of the cost of different technologies. - -""" +"""Harmonize data from Farmer & Lafond (2016) paper on the evolution of the cost of different technologies.""" from owid.catalog import Dataset diff --git a/etl/steps/data/garden/papers/2023-07-10/nemet_2009.py b/etl/steps/data/garden/papers/2023-07-10/nemet_2009.py index 3d53d514a77..7db2e8f937c 100644 --- a/etl/steps/data/garden/papers/2023-07-10/nemet_2009.py +++ b/etl/steps/data/garden/papers/2023-07-10/nemet_2009.py @@ -1,6 +1,4 @@ -"""Harmonize data from Nemet (2009) paper on cost and capacity of photovoltaic energy. - -""" +"""Harmonize data from Nemet (2009) paper on cost and capacity of photovoltaic energy.""" from owid.catalog import Dataset diff --git a/etl/steps/data/garden/papers/2023-12-12/farmer_lafond_2016.py b/etl/steps/data/garden/papers/2023-12-12/farmer_lafond_2016.py index 3cfe424be91..83c38926158 100644 --- a/etl/steps/data/garden/papers/2023-12-12/farmer_lafond_2016.py +++ b/etl/steps/data/garden/papers/2023-12-12/farmer_lafond_2016.py @@ -1,6 +1,4 @@ -"""Harmonize data from Farmer & Lafond (2016) paper on the evolution of the cost of different technologies. - -""" +"""Harmonize data from Farmer & Lafond (2016) paper on the evolution of the cost of different technologies.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/garden/papers/2023-12-12/nemet_2009.py b/etl/steps/data/garden/papers/2023-12-12/nemet_2009.py index 78d22a09113..7a475b4d6fd 100644 --- a/etl/steps/data/garden/papers/2023-12-12/nemet_2009.py +++ b/etl/steps/data/garden/papers/2023-12-12/nemet_2009.py @@ -1,6 +1,4 @@ -"""Harmonize data from Nemet (2009) paper on cost and capacity of photovoltaic energy. - -""" +"""Harmonize data from Nemet (2009) paper on cost and capacity of photovoltaic energy.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/garden/plastic_waste/2023-09-26/geyer_2017.py b/etl/steps/data/garden/plastic_waste/2023-09-26/geyer_2017.py index edd8db37ee3..daabd313501 100644 --- a/etl/steps/data/garden/plastic_waste/2023-09-26/geyer_2017.py +++ b/etl/steps/data/garden/plastic_waste/2023-09-26/geyer_2017.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + import owid.catalog.processing as pr import pandas as pd diff --git a/etl/steps/data/garden/rff/2023-10-19/emissions_weighted_carbon_price.py b/etl/steps/data/garden/rff/2023-10-19/emissions_weighted_carbon_price.py index 547aeeb49e2..0b53791ae39 100644 --- a/etl/steps/data/garden/rff/2023-10-19/emissions_weighted_carbon_price.py +++ b/etl/steps/data/garden/rff/2023-10-19/emissions_weighted_carbon_price.py @@ -1,6 +1,4 @@ -"""Combine dataset on coverage of emissions with the average prices of emissions covered by an ETS or a carbon tax. - -""" +"""Combine dataset on coverage of emissions with the average prices of emissions covered by an ETS or a carbon tax.""" from typing import Set diff --git a/etl/steps/data/garden/rff/2023-10-19/world_carbon_pricing.py b/etl/steps/data/garden/rff/2023-10-19/world_carbon_pricing.py index eb91e913def..7ebd21f510a 100644 --- a/etl/steps/data/garden/rff/2023-10-19/world_carbon_pricing.py +++ b/etl/steps/data/garden/rff/2023-10-19/world_carbon_pricing.py @@ -1,4 +1,5 @@ """Create a garden dataset from the meadow dataset.""" + from typing import Dict, List, cast from owid.catalog import Table, Variable diff --git a/etl/steps/data/garden/shift/2023-07-10/energy_production_from_fossil_fuels.py b/etl/steps/data/garden/shift/2023-07-10/energy_production_from_fossil_fuels.py index 305dd587e9d..749ca76cee2 100644 --- a/etl/steps/data/garden/shift/2023-07-10/energy_production_from_fossil_fuels.py +++ b/etl/steps/data/garden/shift/2023-07-10/energy_production_from_fossil_fuels.py @@ -1,6 +1,4 @@ -"""Garden step for Shift data on energy production from fossil fuels. - -""" +"""Garden step for Shift data on energy production from fossil fuels.""" from typing import Dict, List, Optional, Union diff --git a/etl/steps/data/garden/shift/2023-12-12/energy_production_from_fossil_fuels.py b/etl/steps/data/garden/shift/2023-12-12/energy_production_from_fossil_fuels.py index 1fd2e4a1688..d185a007604 100644 --- a/etl/steps/data/garden/shift/2023-12-12/energy_production_from_fossil_fuels.py +++ b/etl/steps/data/garden/shift/2023-12-12/energy_production_from_fossil_fuels.py @@ -1,6 +1,5 @@ -"""Garden step for Shift data on energy production from fossil fuels. +"""Garden step for Shift data on energy production from fossil fuels.""" -""" import owid.catalog.processing as pr from owid.catalog import Table from structlog import get_logger diff --git a/etl/steps/data/garden/survey/2023-08-04/latinobarometro_trust.py b/etl/steps/data/garden/survey/2023-08-04/latinobarometro_trust.py index f196269ff5d..f814f424cfe 100644 --- a/etl/steps/data/garden/survey/2023-08-04/latinobarometro_trust.py +++ b/etl/steps/data/garden/survey/2023-08-04/latinobarometro_trust.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - from owid.catalog import Table from etl.data_helpers import geo diff --git a/etl/steps/data/garden/survey/2023-08-07/afrobarometer_trust.py b/etl/steps/data/garden/survey/2023-08-07/afrobarometer_trust.py index 88e746f1012..3c8b916a5f2 100644 --- a/etl/steps/data/garden/survey/2023-08-07/afrobarometer_trust.py +++ b/etl/steps/data/garden/survey/2023-08-07/afrobarometer_trust.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - from owid.catalog import Table from etl.data_helpers import geo diff --git a/etl/steps/data/garden/tuberculosis/2023-11-27/drug_resistance_surveillance.py b/etl/steps/data/garden/tuberculosis/2023-11-27/drug_resistance_surveillance.py index 8a50d450ee0..57f2dd9a347 100644 --- a/etl/steps/data/garden/tuberculosis/2023-11-27/drug_resistance_surveillance.py +++ b/etl/steps/data/garden/tuberculosis/2023-11-27/drug_resistance_surveillance.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + import numpy as np from owid.catalog import Table from shared import add_variable_description_from_producer diff --git a/etl/steps/data/garden/tuberculosis/2023-11-27/notifications.py b/etl/steps/data/garden/tuberculosis/2023-11-27/notifications.py index cb01a28182b..11eecd1d8e6 100644 --- a/etl/steps/data/garden/tuberculosis/2023-11-27/notifications.py +++ b/etl/steps/data/garden/tuberculosis/2023-11-27/notifications.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + from owid.catalog import Table from owid.catalog import processing as pr from shared import add_variable_description_from_producer, removing_old_variables diff --git a/etl/steps/data/garden/tuberculosis/2023-11-27/outcomes.py b/etl/steps/data/garden/tuberculosis/2023-11-27/outcomes.py index 5c4ec33298a..3d8cd902f24 100644 --- a/etl/steps/data/garden/tuberculosis/2023-11-27/outcomes.py +++ b/etl/steps/data/garden/tuberculosis/2023-11-27/outcomes.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + from shared import add_variable_description_from_producer, removing_old_variables from etl.data_helpers import geo diff --git a/etl/steps/data/garden/tuberculosis/2023-11-27/unhlm_commitments.py b/etl/steps/data/garden/tuberculosis/2023-11-27/unhlm_commitments.py index 020b033f783..28eb5b1d734 100644 --- a/etl/steps/data/garden/tuberculosis/2023-11-27/unhlm_commitments.py +++ b/etl/steps/data/garden/tuberculosis/2023-11-27/unhlm_commitments.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + from owid.catalog import Table from shared import add_variable_description_from_producer diff --git a/etl/steps/data/garden/un/2022-07-11/un_wpp/deaths.py b/etl/steps/data/garden/un/2022-07-11/un_wpp/deaths.py index 9d69fdf2854..0a22b53dd40 100644 --- a/etl/steps/data/garden/un/2022-07-11/un_wpp/deaths.py +++ b/etl/steps/data/garden/un/2022-07-11/un_wpp/deaths.py @@ -1,4 +1,5 @@ """Deaths table""" + from typing import Any, Dict import pandas as pd diff --git a/etl/steps/data/garden/un/2022-07-11/un_wpp/demographics.py b/etl/steps/data/garden/un/2022-07-11/un_wpp/demographics.py index e0c7ec933bc..57472b65a42 100644 --- a/etl/steps/data/garden/un/2022-07-11/un_wpp/demographics.py +++ b/etl/steps/data/garden/un/2022-07-11/un_wpp/demographics.py @@ -1,4 +1,5 @@ """Demographics table.""" + from typing import Any, Dict from owid.catalog import Table diff --git a/etl/steps/data/garden/un/2022-07-11/un_wpp/dep_ratio.py b/etl/steps/data/garden/un/2022-07-11/un_wpp/dep_ratio.py index 05ff417c714..4e9f01c7dda 100644 --- a/etl/steps/data/garden/un/2022-07-11/un_wpp/dep_ratio.py +++ b/etl/steps/data/garden/un/2022-07-11/un_wpp/dep_ratio.py @@ -1,4 +1,5 @@ """Depenndency ratio table""" + from owid.catalog import Table from .dtypes import optimize_dtypes diff --git a/etl/steps/data/garden/un/2022-07-11/un_wpp/fertility.py b/etl/steps/data/garden/un/2022-07-11/un_wpp/fertility.py index a68281e3f60..f2a79b93493 100644 --- a/etl/steps/data/garden/un/2022-07-11/un_wpp/fertility.py +++ b/etl/steps/data/garden/un/2022-07-11/un_wpp/fertility.py @@ -1,4 +1,5 @@ """Fertility table.""" + from typing import Any, Dict, List from owid.catalog import Table diff --git a/etl/steps/data/garden/un/2022-07-11/un_wpp/population.py b/etl/steps/data/garden/un/2022-07-11/un_wpp/population.py index f046595af36..cdc8122967f 100644 --- a/etl/steps/data/garden/un/2022-07-11/un_wpp/population.py +++ b/etl/steps/data/garden/un/2022-07-11/un_wpp/population.py @@ -1,4 +1,5 @@ """Population table.""" + from typing import Any, Dict, List, Optional, Tuple import numpy as np diff --git a/etl/steps/data/garden/un/2023-08-16/igme.py b/etl/steps/data/garden/un/2023-08-16/igme.py index 1cee5cdc8a9..054652e972d 100644 --- a/etl/steps/data/garden/un/2023-08-16/igme.py +++ b/etl/steps/data/garden/un/2023-08-16/igme.py @@ -383,13 +383,13 @@ def fix_sub_saharan_africa(tb: Table) -> Table: """ tb["country"] = tb["country"].astype(str) - tb.loc[ - (tb["country"] == "Sub-Saharan Africa") & (tb["regional_group"] == "UNICEF"), "country" - ] = "Sub-Saharan Africa (UNICEF)" + tb.loc[(tb["country"] == "Sub-Saharan Africa") & (tb["regional_group"] == "UNICEF"), "country"] = ( + "Sub-Saharan Africa (UNICEF)" + ) - tb.loc[ - (tb["country"] == "Sub-Saharan Africa") & (tb["regional_group"] == "SDG"), "country" - ] = "Sub-Saharan Africa (SDG)" + tb.loc[(tb["country"] == "Sub-Saharan Africa") & (tb["regional_group"] == "SDG"), "country"] = ( + "Sub-Saharan Africa (SDG)" + ) return tb diff --git a/etl/steps/data/garden/un/2023-08-29/long_run_child_mortality.py b/etl/steps/data/garden/un/2023-08-29/long_run_child_mortality.py index 86f364593e3..d88afde9229 100644 --- a/etl/steps/data/garden/un/2023-08-29/long_run_child_mortality.py +++ b/etl/steps/data/garden/un/2023-08-29/long_run_child_mortality.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + import owid.catalog.processing as pr from owid.catalog import Dataset, Table diff --git a/etl/steps/data/garden/un/2024-01-17/urban_agglomerations_300k.py b/etl/steps/data/garden/un/2024-01-17/urban_agglomerations_300k.py index 92d6d4886a3..1b93e5e9c5a 100644 --- a/etl/steps/data/garden/un/2024-01-17/urban_agglomerations_300k.py +++ b/etl/steps/data/garden/un/2024-01-17/urban_agglomerations_300k.py @@ -50,15 +50,15 @@ def run(dest_dir: str) -> None: } ) - # Create two new dataframes to separate data into estimates and projections (pre-2019 and post-2019) - past_estimates = tb_average[tb_average["year"] < 2019].copy() - future_projections = tb_average[tb_average["year"] >= 2019].copy() + # Create two new dataframes to separate data into estimates and projections + past_estimates = tb_average[tb_average["year"] <= 2015].copy() + future_projections = tb_average[tb_average["year"] >= 2015].copy() # Now, for each column in the original dataframe, split it into two for col in tb_average.columns: if col not in ["country", "year"]: - past_estimates[f"{col}_estimates"] = tb_average.loc[tb_average["year"] < 2019, col] - future_projections[f"{col}_projections"] = tb_average.loc[tb_average["year"] >= 2019, col] + past_estimates[f"{col}_estimates"] = tb_average.loc[tb_average["year"] <= 2015, col] + future_projections[f"{col}_projections"] = tb_average.loc[tb_average["year"] >= 2015, col] past_estimates = past_estimates.drop(columns=[col]) future_projections = future_projections.drop(columns=[col]) diff --git a/etl/steps/data/garden/un/2024-01-17/urban_agglomerations_definition_count.py b/etl/steps/data/garden/un/2024-01-17/urban_agglomerations_definition_count.py index 66c12efbfd0..324e0bd4783 100644 --- a/etl/steps/data/garden/un/2024-01-17/urban_agglomerations_definition_count.py +++ b/etl/steps/data/garden/un/2024-01-17/urban_agglomerations_definition_count.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - from owid.catalog import Table from structlog import get_logger diff --git a/etl/steps/data/garden/un/2024-01-17/urban_agglomerations_size_class.py b/etl/steps/data/garden/un/2024-01-17/urban_agglomerations_size_class.py index 6aa4daf3d05..e44531c7a8a 100644 --- a/etl/steps/data/garden/un/2024-01-17/urban_agglomerations_size_class.py +++ b/etl/steps/data/garden/un/2024-01-17/urban_agglomerations_size_class.py @@ -38,15 +38,16 @@ def run(dest_dir: str) -> None: tb_pivot[col] = tb_pivot[col] * 1000 tb_pivot = tb_pivot.reset_index() - # Create two new dataframes to separate data into estimates and projections (pre-2019 and post-2019) - past_estimates = tb_pivot[tb_pivot["year"] < 2019].copy() - future_projections = tb_pivot[tb_pivot["year"] >= 2019].copy() + # Create two new dataframes to separate data into estimates and projections + + past_estimates = tb_pivot[tb_pivot["year"] <= 2015].copy() + future_projections = tb_pivot[tb_pivot["year"] >= 2015].copy() # Now, for each column in the original dataframe, split it into two (projections and estimates) for col in tb_pivot.columns: if col not in ["country", "year"]: - past_estimates[f"{col}_estimates"] = tb_pivot.loc[tb_pivot["year"] < 2019, col] - future_projections[f"{col}_projections"] = tb_pivot.loc[tb_pivot["year"] >= 2019, col] + past_estimates[f"{col}_estimates"] = tb_pivot.loc[tb_pivot["year"] <= 2015, col] + future_projections[f"{col}_projections"] = tb_pivot.loc[tb_pivot["year"] >= 2015, col] past_estimates = past_estimates.drop(columns=[col]) future_projections = future_projections.drop(columns=[col]) diff --git a/etl/steps/data/garden/un/2024-01-17/urbanization_urban_rural.py b/etl/steps/data/garden/un/2024-01-17/urbanization_urban_rural.py index c22b5c9aef1..9189fee2081 100644 --- a/etl/steps/data/garden/un/2024-01-17/urbanization_urban_rural.py +++ b/etl/steps/data/garden/un/2024-01-17/urbanization_urban_rural.py @@ -48,15 +48,15 @@ def run(dest_dir: str) -> None: # Remove 'thousands' from column name tb.rename(columns={col: col.replace("__thousands", "")}, inplace=True) - # Create two new dataframes to separate data into estimates and projections (pre-2019 and post-2019) - past_estimates = tb[tb["year"] < 2019].copy() - future_projections = tb[tb["year"] >= 2019].copy() + # Create two new dataframes to separate data into estimates and projections (pre-2015 and post-2015) + past_estimates = tb[tb["year"] <= 2015].copy() + future_projections = tb[tb["year"] >= 2015].copy() # Now, for each column in the original dataframe, split it into two (projections and estimates) for col in tb.columns: if col not in ["country", "year"]: - past_estimates[f"{col}_estimates"] = tb.loc[tb["year"] < 2019, col] - future_projections[f"{col}_projections"] = tb.loc[tb["year"] >= 2019, col] + past_estimates[f"{col}_estimates"] = tb.loc[tb["year"] <= 2015, col] + future_projections[f"{col}_projections"] = tb.loc[tb["year"] >= 2015, col] past_estimates = past_estimates.drop(columns=[col]) future_projections = future_projections.drop(columns=[col]) diff --git a/etl/steps/data/garden/un/2024-02-14/sdgs_urbanization.py b/etl/steps/data/garden/un/2024-02-14/sdgs_urbanization.py index 7851d54e94b..3c602479aa3 100644 --- a/etl/steps/data/garden/un/2024-02-14/sdgs_urbanization.py +++ b/etl/steps/data/garden/un/2024-02-14/sdgs_urbanization.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - from etl.data_helpers import geo from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/garden/un/2024-03-14/un_wpp_most.meta.yml b/etl/steps/data/garden/un/2024-03-14/un_wpp_most.meta.yml index c1e97dd6da2..c7793d9cbb2 100644 --- a/etl/steps/data/garden/un/2024-03-14/un_wpp_most.meta.yml +++ b/etl/steps/data/garden/un/2024-03-14/un_wpp_most.meta.yml @@ -1,13 +1,21 @@ +definitions: + common: + presentation: + grapher_config: + subtitle: "" + originUrl: "https://ourworldindata.org/population-growth" + note: "" + tables: population_5_year_age_groups: variables: - age: + age_group_five: title: Five year age-group with the highest population unit: "" description_short: |- Five-year age group with the highest population. type: ordinal - sort: + sort: # May need additional groups when data is updated - 0-4 - 5-9 - 10-14 @@ -23,22 +31,18 @@ tables: - 60-64 - 65-69 - 70-74 - value: - title: Population of the most populous five-year age group - unit: "people" - description_short: |- - Population of the most populous five-year age-group. - display: - numDecimalPlaces: 0 + - 75-79 + presentation: + title_public: Five year age-group with the highest population population_10_year_age_groups: variables: - age_group: + age_group_ten: title: Ten year age-group with the highest population unit: "" description_short: |- Ten-year age group with the highest population. type: ordinal - sort: + sort: # May need additional groups when data is updated - 0-9 - 10-19 - 20-29 @@ -46,10 +50,6 @@ tables: - 40-49 - 50-59 - 60-69 - value: - title: Population of the most populous ten year age group - unit: "people" - description_short: |- - Population of the most populous ten-year age-group. - display: - numDecimalPlaces: 0 + - 70-79 + presentation: + title_public: Ten year age-group with the highest population diff --git a/etl/steps/data/garden/un/2024-03-14/un_wpp_most.py b/etl/steps/data/garden/un/2024-03-14/un_wpp_most.py index c067d01d25e..0b0220c589f 100644 --- a/etl/steps/data/garden/un/2024-03-14/un_wpp_most.py +++ b/etl/steps/data/garden/un/2024-03-14/un_wpp_most.py @@ -1,10 +1,14 @@ +from typing import Any + from owid.catalog import Table from owid.catalog import processing as pr +from structlog import get_logger from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. paths = PathFinder(__file__) +log = get_logger() def run(dest_dir: str) -> None: @@ -12,27 +16,31 @@ def run(dest_dir: str) -> None: # Load inputs. ds_garden = paths.load_dataset("un_wpp") tb_pop = ds_garden["population"].reset_index() - + origins = tb_pop["population"].metadata.origins[0] age_group_size = [5, 10] tb_list = [] tb_pop_filter = Table() for age_group in age_group_size: + log.info(f"Creating population table for {age_group} year age groups") # filter data for just sex = all, metrics = population, variant = estimates if age_group == 5: - tb_pop_filter = create_five_year_age_groups(tb_pop) + tb_pop_filter = create_five_year_age_groups(tb_pop, origins) if age_group == 10: - tb_pop_filter = create_ten_year_age_groups(tb_pop) + tb_pop_filter = create_ten_year_age_groups(tb_pop, origins) # Group by country and year, and apply the custom function - tb_pop_filter = tb_pop_filter.groupby(["location", "year"], observed=False).apply(get_largest_age_group) - # The function above creates NAs for some locations that don't appear to be in the table e.g. Vatican, Melanesia, so dropping here - tb_pop_filter = tb_pop_filter.dropna() - tb_pop_filter = tb_pop_filter.reset_index(drop=True) - tb_pop_filter = tb_pop_filter.set_index(["location", "year"], verify_integrity=True) - tb_pop_filter = tb_pop_filter.copy_metadata(tb_pop) + tb_pop_filter = ( + tb_pop_filter.groupby(["country", "year"], group_keys=False) + .apply(get_largest_age_group) + .reset_index(drop=True) # Reset index to have a clean DataFrame + ) + # The function above creates NAs for some countrys that don't appear to be in the table e.g. Vatican, Melanesia, so dropping here + + tb_pop_filter = tb_pop_filter.drop(columns=["population"]) + tb_pop_filter = tb_pop_filter.set_index(["country", "year"], verify_integrity=True) tb_pop_filter.metadata.short_name = f"population_{age_group}_year_age_groups" tb_list.append(tb_pop_filter) # Save outputs. - # + # Create a new garden dataset with the same metadata as the meadow dataset. ds_garden = create_dataset(dest_dir, tables=tb_list, default_metadata=ds_garden.metadata) @@ -40,7 +48,7 @@ def run(dest_dir: str) -> None: ds_garden.save() -def create_ten_year_age_groups(tb: Table) -> Table: +def create_ten_year_age_groups(tb: Table, origins: Any) -> Table: # Initialize an empty list to hold the age bands age_bands = [] # Loop through a range with a step of 5, stopping before 100 @@ -49,28 +57,29 @@ def create_ten_year_age_groups(tb: Table) -> Table: # Add the "100+" group at the end and 0-4 and 5-9 as 0-9 is not a group in the dataset age_bands = age_bands + ["100+", "0-4", "5-9", "10-14", "15-19"] # Filter the table to only include the age bands we want - tb = tb[(tb.sex == "all") & (tb.metric == "population") & (tb.variant == "estimates") & (tb.age.isin(age_bands))] + tb = tb[(tb.sex == "all") & (tb.variant == "estimates") & (tb.age.isin(age_bands))] assert tb["age"].nunique() == len(age_bands), "Age groups are not as expected" - tb = tb.drop(columns=["metric", "sex", "variant"]) + tb = tb.drop(columns=["sex", "variant", "population_change", "population_density"]) # Create the 0-9 and 10-19 age groups tb_0_9 = tb[(tb.age == "0-4") | (tb.age == "5-9")] - tb_0_9 = tb_0_9.groupby(["location", "year"], observed=False)["value"].sum().reset_index() + tb_0_9 = tb_0_9.groupby(["country", "year"], observed=False)["population"].sum().reset_index() tb_0_9["age"] = "0-9" tb_10_19 = tb[(tb.age == "10-14") | (tb.age == "15-19")] - tb_10_19 = tb_10_19.groupby(["location", "year"], observed=False)["value"].sum().reset_index() + tb_10_19 = tb_10_19.groupby(["country", "year"], observed=False)["population"].sum().reset_index() tb_10_19["age"] = "10-19" # Drop the 0-4, 5-9, 10-14 and 15-19 age groups tb = tb[(tb.age != "0-4") & (tb.age != "5-9") & (tb.age != "10-14") & (tb.age != "15-19")] # Concatenate the 0-9 and 10-19 age groups with the original table tb = pr.concat([tb, tb_0_9, tb_10_19]) - tb = tb.rename(columns={"age": "age_group"}) + tb = tb.rename(columns={"age": "age_group_ten"}) + tb["age_group_ten"].metadata.origins = [origins] tb = tb.reset_index(drop=True) return tb -def create_five_year_age_groups(tb: Table) -> Table: +def create_five_year_age_groups(tb: Table, origins: Any) -> Table: # Initialize an empty list to hold the age bands age_bands = [] # Loop through a range with a step of 5, stopping before 100 @@ -79,13 +88,15 @@ def create_five_year_age_groups(tb: Table) -> Table: # Add the "100+" group at the end age_bands.append("100+") # Filter the table to only include the age bands we want - tb = tb[(tb.sex == "all") & (tb.metric == "population") & (tb.variant == "estimates") & (tb.age.isin(age_bands))] + tb = tb[(tb.sex == "all") & (tb.variant == "estimates") & (tb.age.isin(age_bands))] assert tb["age"].nunique() == len(age_bands), "Age groups are not as expected" - tb = tb.drop(columns=["metric", "sex", "variant"]) + tb = tb.drop(columns=["sex", "variant", "population_change", "population_density"]) + tb = tb.rename(columns={"age": "age_group_five"}) + tb["age_group_five"].metadata.origins = [origins] tb = tb.reset_index(drop=True) return tb # Function to apply to each group to find the age group with the largest population def get_largest_age_group(group): - return group.loc[group["value"].idxmax()] + return group.loc[group["population"].idxmax()] diff --git a/etl/steps/data/garden/un/2024-07-11/un_wpp.meta.yml b/etl/steps/data/garden/un/2024-07-11/un_wpp.meta.yml index 8097420beb4..1344ff48672 100644 --- a/etl/steps/data/garden/un/2024-07-11/un_wpp.meta.yml +++ b/etl/steps/data/garden/un/2024-07-11/un_wpp.meta.yml @@ -1,7 +1,7 @@ definitions: global: projections: - <%- if variant != 'estimates' -%> + <%- if (variant is defined) and (variant != 'estimates') -%> Projections from 2024 onwards are based on the UN's << variant >> scenario. <%- endif -%> dimensions: @@ -39,6 +39,7 @@ definitions: {definitions.global.projections} description_short_births: |- + <%- if sex is defined -%> <%- if not (sex == 'all' and age == 'all') -%> This only includes <%- if sex != 'all' -%> @@ -48,6 +49,7 @@ definitions: mothers aged << age >> <%- endif -%>. <%- endif -%> + <%- endif -%> {definitions.global.projections} diff --git a/etl/steps/data/garden/un/2024-07-12/un_wpp.meta.yml b/etl/steps/data/garden/un/2024-07-12/un_wpp.meta.yml index c62b1eac1fe..3d59ea4fcb8 100644 --- a/etl/steps/data/garden/un/2024-07-12/un_wpp.meta.yml +++ b/etl/steps/data/garden/un/2024-07-12/un_wpp.meta.yml @@ -1,7 +1,7 @@ definitions: global: projections: - <%- if variant != 'estimates' -%> + <%- if (variant is defined) and (variant != 'estimates') -%> Projections from 2024 onwards are based on the UN's << variant >> scenario. <%- endif -%> dimensions: @@ -43,6 +43,7 @@ definitions: {definitions.global.projections} description_short_births: |- + <%- if sex is defined -%> <%-if not (sex == 'all' and age == 'all') -%> This only includes <%- if sex != 'all' -%> @@ -51,7 +52,8 @@ definitions: <%- if age != 'all' -%> mothers aged << age >> <%- endif -%>. - <%-endif -%> + <%- endif -%> + <%- endif -%> {definitions.global.projections} diff --git a/etl/steps/data/garden/un/2024-07-25/resettlement.py b/etl/steps/data/garden/un/2024-07-25/resettlement.py index 0e9ce903186..51959cfd58e 100644 --- a/etl/steps/data/garden/un/2024-07-25/resettlement.py +++ b/etl/steps/data/garden/un/2024-07-25/resettlement.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + import pandas as pd from owid.catalog import processing as pr diff --git a/etl/steps/data/garden/un/2024-09-11/igme.meta.yml b/etl/steps/data/garden/un/2024-09-11/igme.meta.yml index 74ecc8ac14f..b2c70afefec 100644 --- a/etl/steps/data/garden/un/2024-09-11/igme.meta.yml +++ b/etl/steps/data/garden/un/2024-09-11/igme.meta.yml @@ -32,6 +32,7 @@ definitions: <%- endif -%> indicator: display_name: |- + <% set indicator = indicator | default('') %> <% if indicator == "Neonatal mortality rate" %> The estimated number of deaths of {definitions.sex} children aged less than 28 days, per 100 live births, in {definitions.wealth_quintile} <% elif indicator == "Mortality rate 1-59 months" %> @@ -102,43 +103,43 @@ tables: title: Observation value unit: |- {definitions.unit_of_measure.unit} - short_unit: <% if 'rate' in indicator %>%<% else %><%- endif -%> + short_unit: <% if 'rate' in indicator | default('') %>%<% else %><%- endif -%> display: name: |- {definitions.indicator.display_name} - numDecimalPlaces: <% if 'rate' in indicator %>1<% else %>0<%- endif -%> + numDecimalPlaces: <% if 'rate' in indicator | default('') %>1<% else %>0<%- endif -%> presentation: attribution: United Nations Inter-agency Group for Child Mortality Estimation (2024) lower_bound: title: Lower bound unit: |- {definitions.unit_of_measure.unit} - short_unit: <% if 'rate' in indicator %>%<% else %><%- endif -%> + short_unit: <% if 'rate' in indicator | default('') %>%<% else %><%- endif -%> display: name: |- {definitions.indicator.display_name} - numDecimalPlaces: <% if 'rate' in indicator %>1<% else %>0<%- endif -%> + numDecimalPlaces: <% if 'rate' in indicator | default('') %>1<% else %>0<%- endif -%> presentation: attribution: United Nations Inter-agency Group for Child Mortality Estimation (2024) upper_bound: title: Upper bound unit: |- {definitions.unit_of_measure.unit} - short_unit: <% if 'rate' in indicator %>%<% else %><%- endif -%> + short_unit: <% if 'rate' in indicator | default('') %>%<% else %><%- endif -%> display: name: |- {definitions.indicator.display_name} - numDecimalPlaces: <% if 'rate' in indicator %>1<% else %>0<%- endif -%> + numDecimalPlaces: <% if 'rate' in indicator | default('') %>1<% else %>0<%- endif -%> presentation: attribution: United Nations Inter-agency Group for Child Mortality Estimation (2024) igme_under_fifteen_mortality: variables: obs_value: title: <> - description_short: <% if 'rate' in indicator %>Estimated share of children that die before reaching the age of fifteen.<% else %>Estiamted number of children that die before reaching the age of fifteen.<%- endif -%> + description_short: <% if 'rate' in indicator | default('') %>Estimated share of children that die before reaching the age of fifteen.<% else %>Estiamted number of children that die before reaching the age of fifteen.<%- endif -%> description_processing: This indicator is processed by OWID based on the original data source. It is a combination of the under-five mortality rate and the 5-14 mortality rate. unit: <> - short_unit: <% if 'rate' in indicator %>%<% else %><%- endif -%> + short_unit: <% if 'rate' in indicator | default('') %>%<% else %><%- endif -%> display: name: <> numDecimalPlaces: <% if 'rate' in indicator %>2<% else %>0<%- endif -%> diff --git a/etl/steps/data/garden/un/2024-09-11/igme.py b/etl/steps/data/garden/un/2024-09-11/igme.py index b74d410f6d2..27450198a6d 100644 --- a/etl/steps/data/garden/un/2024-09-11/igme.py +++ b/etl/steps/data/garden/un/2024-09-11/igme.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + from math import trunc from typing import List diff --git a/etl/steps/data/garden/un/2024-09-16/long_run_child_mortality.py b/etl/steps/data/garden/un/2024-09-16/long_run_child_mortality.py index 52e841fe4a2..778ebea754c 100644 --- a/etl/steps/data/garden/un/2024-09-16/long_run_child_mortality.py +++ b/etl/steps/data/garden/un/2024-09-16/long_run_child_mortality.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + import owid.catalog.processing as pr from owid.catalog import Table diff --git a/etl/steps/data/garden/un/2024-12-02/un_wpp_lt.countries.json b/etl/steps/data/garden/un/2024-12-02/un_wpp_lt.countries.json new file mode 100644 index 00000000000..2eccd289241 --- /dev/null +++ b/etl/steps/data/garden/un/2024-12-02/un_wpp_lt.countries.json @@ -0,0 +1,254 @@ +{ + "Afghanistan": "Afghanistan", + "Albania": "Albania", + "Algeria": "Algeria", + "American Samoa": "American Samoa", + "Andorra": "Andorra", + "Angola": "Angola", + "Anguilla": "Anguilla", + "Antigua and Barbuda": "Antigua and Barbuda", + "Argentina": "Argentina", + "Armenia": "Armenia", + "Aruba": "Aruba", + "Australia": "Australia", + "Austria": "Austria", + "Azerbaijan": "Azerbaijan", + "Bahamas": "Bahamas", + "Bahrain": "Bahrain", + "Bangladesh": "Bangladesh", + "Barbados": "Barbados", + "Belarus": "Belarus", + "Belgium": "Belgium", + "Belize": "Belize", + "Benin": "Benin", + "Bermuda": "Bermuda", + "Bhutan": "Bhutan", + "Bolivia (Plurinational State of)": "Bolivia", + "Bonaire, Sint Eustatius and Saba": "Bonaire Sint Eustatius and Saba", + "Bosnia and Herzegovina": "Bosnia and Herzegovina", + "Botswana": "Botswana", + "Brazil": "Brazil", + "British Virgin Islands": "British Virgin Islands", + "Brunei Darussalam": "Brunei", + "Bulgaria": "Bulgaria", + "Burkina Faso": "Burkina Faso", + "Burundi": "Burundi", + "Cabo Verde": "Cape Verde", + "Cambodia": "Cambodia", + "Cameroon": "Cameroon", + "Canada": "Canada", + "Cayman Islands": "Cayman Islands", + "Central African Republic": "Central African Republic", + "Chad": "Chad", + "Chile": "Chile", + "China": "China", + "China, Hong Kong SAR": "Hong Kong", + "China, Macao SAR": "Macao", + "Colombia": "Colombia", + "Comoros": "Comoros", + "Congo": "Congo", + "Cook Islands": "Cook Islands", + "Costa Rica": "Costa Rica", + "Croatia": "Croatia", + "Cuba": "Cuba", + "Cura\u00e7ao": "Curacao", + "Cyprus": "Cyprus", + "Czechia": "Czechia", + "C\u00f4te d'Ivoire": "Cote d'Ivoire", + "Democratic Republic of the Congo": "Democratic Republic of Congo", + "Denmark": "Denmark", + "Djibouti": "Djibouti", + "Dominica": "Dominica", + "Dominican Republic": "Dominican Republic", + "Ecuador": "Ecuador", + "Egypt": "Egypt", + "El Salvador": "El Salvador", + "Equatorial Guinea": "Equatorial Guinea", + "Eritrea": "Eritrea", + "Estonia": "Estonia", + "Eswatini": "Eswatini", + "Ethiopia": "Ethiopia", + "Falkland Islands (Malvinas)": "Falkland Islands", + "Faroe Islands": "Faroe Islands", + "Fiji": "Fiji", + "Finland": "Finland", + "France": "France", + "French Guiana": "French Guiana", + "French Polynesia": "French Polynesia", + "Gabon": "Gabon", + "Gambia": "Gambia", + "Georgia": "Georgia", + "Germany": "Germany", + "Ghana": "Ghana", + "Gibraltar": "Gibraltar", + "Greece": "Greece", + "Greenland": "Greenland", + "Grenada": "Grenada", + "Guadeloupe": "Guadeloupe", + "Guam": "Guam", + "Guatemala": "Guatemala", + "Guernsey": "Guernsey", + "Guinea": "Guinea", + "Guinea-Bissau": "Guinea-Bissau", + "Guyana": "Guyana", + "Haiti": "Haiti", + "Honduras": "Honduras", + "Hungary": "Hungary", + "Iceland": "Iceland", + "India": "India", + "Indonesia": "Indonesia", + "Iran (Islamic Republic of)": "Iran", + "Iraq": "Iraq", + "Ireland": "Ireland", + "Isle of Man": "Isle of Man", + "Israel": "Israel", + "Italy": "Italy", + "Jamaica": "Jamaica", + "Japan": "Japan", + "Jersey": "Jersey", + "Jordan": "Jordan", + "Kazakhstan": "Kazakhstan", + "Kenya": "Kenya", + "Kiribati": "Kiribati", + "Kuwait": "Kuwait", + "Kyrgyzstan": "Kyrgyzstan", + "Lao People's Democratic Republic": "Laos", + "Latvia": "Latvia", + "Lebanon": "Lebanon", + "Lesotho": "Lesotho", + "Liberia": "Liberia", + "Libya": "Libya", + "Liechtenstein": "Liechtenstein", + "Lithuania": "Lithuania", + "Luxembourg": "Luxembourg", + "Madagascar": "Madagascar", + "Malawi": "Malawi", + "Malaysia": "Malaysia", + "Maldives": "Maldives", + "Mali": "Mali", + "Malta": "Malta", + "Marshall Islands": "Marshall Islands", + "Martinique": "Martinique", + "Mauritania": "Mauritania", + "Mauritius": "Mauritius", + "Mayotte": "Mayotte", + "Melanesia": "Melanesia", + "Mexico": "Mexico", + "Micronesia": "Micronesia (country)", + "Micronesia (Fed. States of)": "Micronesia (country)", + "Monaco": "Monaco", + "Mongolia": "Mongolia", + "Montenegro": "Montenegro", + "Montserrat": "Montserrat", + "Morocco": "Morocco", + "Mozambique": "Mozambique", + "Myanmar": "Myanmar", + "Namibia": "Namibia", + "Nauru": "Nauru", + "Nepal": "Nepal", + "Netherlands": "Netherlands", + "New Caledonia": "New Caledonia", + "New Zealand": "New Zealand", + "Nicaragua": "Nicaragua", + "Niger": "Niger", + "Nigeria": "Nigeria", + "Niue": "Niue", + "North Macedonia": "North Macedonia", + "Northern Mariana Islands": "Northern Mariana Islands", + "Norway": "Norway", + "Oman": "Oman", + "Pakistan": "Pakistan", + "Palau": "Palau", + "Panama": "Panama", + "Papua New Guinea": "Papua New Guinea", + "Paraguay": "Paraguay", + "Peru": "Peru", + "Philippines": "Philippines", + "Poland": "Poland", + "Polynesia": "Polynesia", + "Portugal": "Portugal", + "Puerto Rico": "Puerto Rico", + "Qatar": "Qatar", + "Republic of Korea": "South Korea", + "Republic of Moldova": "Moldova", + "Romania": "Romania", + "Russian Federation": "Russia", + "Rwanda": "Rwanda", + "R\u00e9union": "Reunion", + "Saint Barth\u00e9lemy": "Saint Barthelemy", + "Saint Helena": "Saint Helena", + "Saint Kitts and Nevis": "Saint Kitts and Nevis", + "Saint Lucia": "Saint Lucia", + "Saint Martin (French part)": "Saint Martin (French part)", + "Saint Pierre and Miquelon": "Saint Pierre and Miquelon", + "Saint Vincent and the Grenadines": "Saint Vincent and the Grenadines", + "Samoa": "Samoa", + "San Marino": "San Marino", + "Sao Tome and Principe": "Sao Tome and Principe", + "Saudi Arabia": "Saudi Arabia", + "Senegal": "Senegal", + "Serbia": "Serbia", + "Seychelles": "Seychelles", + "Sierra Leone": "Sierra Leone", + "Singapore": "Singapore", + "Sint Maarten (Dutch part)": "Sint Maarten (Dutch part)", + "Slovakia": "Slovakia", + "Slovenia": "Slovenia", + "Solomon Islands": "Solomon Islands", + "Somalia": "Somalia", + "South Africa": "South Africa", + "South America": "South America", + "South Sudan": "South Sudan", + "Spain": "Spain", + "Sri Lanka": "Sri Lanka", + "State of Palestine": "Palestine", + "Sudan": "Sudan", + "Suriname": "Suriname", + "Sweden": "Sweden", + "Switzerland": "Switzerland", + "Syrian Arab Republic": "Syria", + "Tajikistan": "Tajikistan", + "Thailand": "Thailand", + "Timor-Leste": "East Timor", + "Togo": "Togo", + "Tokelau": "Tokelau", + "Tonga": "Tonga", + "Trinidad and Tobago": "Trinidad and Tobago", + "Tunisia": "Tunisia", + "Turkmenistan": "Turkmenistan", + "Turks and Caicos Islands": "Turks and Caicos Islands", + "Tuvalu": "Tuvalu", + "Uganda": "Uganda", + "Ukraine": "Ukraine", + "United Arab Emirates": "United Arab Emirates", + "United Kingdom": "United Kingdom", + "United Republic of Tanzania": "Tanzania", + "United States Virgin Islands": "United States Virgin Islands", + "United States of America": "United States", + "Uruguay": "Uruguay", + "Uzbekistan": "Uzbekistan", + "Vanuatu": "Vanuatu", + "Venezuela (Bolivarian Republic of)": "Venezuela", + "Viet Nam": "Vietnam", + "Wallis and Futuna Islands": "Wallis and Futuna", + "Western Sahara": "Western Sahara", + "World": "World", + "Yemen": "Yemen", + "Zambia": "Zambia", + "Zimbabwe": "Zimbabwe", + "China, Taiwan Province of China": "Taiwan", + "Dem. People's Republic of Korea": "North Korea", + "Kosovo (under UNSC res. 1244)": "Kosovo", + "T\u00fcrkiye": "Turkey", + "High-income countries": "High-income countries", + "Low-income countries": "Low-income countries", + "Lower-middle-income countries": "Lower-middle-income countries", + "Upper-middle-income countries": "Upper-middle-income countries", + "Latin America and the Caribbean": "Latin America and the Caribbean (UN)", + "Africa": "Africa (UN)", + "Asia": "Asia (UN)", + "Europe": "Europe (UN)", + "Northern America": "Northern America (UN)", + "Oceania": "Oceania (UN)", + "Holy See": "Vatican" +} diff --git a/etl/steps/data/garden/un/2024-12-02/un_wpp_lt.meta.yml b/etl/steps/data/garden/un/2024-12-02/un_wpp_lt.meta.yml new file mode 100644 index 00000000000..c7b3d363bb8 --- /dev/null +++ b/etl/steps/data/garden/un/2024-12-02/un_wpp_lt.meta.yml @@ -0,0 +1,104 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + life_expectancy: + description_short: |- + <%- if type == "period" -%> + Period life expectancy is the average number of years a person is expected to live, based on mortality rates seen across all age groups in a given interval. + <%- elif type == "cohort" -%> + Cohort life expectancy is the average number of years that individuals born in a given interval actually lived, based on tracking that birth cohort over time. + <%- endif -%> + point_1: |- + <%- if type == "period" -%> + This is based on a synthetic cohort created using mortality rates across age groups in a given year. + <%- elif type == "cohort" -%> + Rather than waiting for the entire cohort to have died before calculating the cohort life expectancy, researchers may use data from birth cohorts that are 'almost extinct'. + <%- endif -%> + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/dataset/ +dataset: + update_period_days: 365 + description: |- + Abridged life tables up to age 100 by sex and both sexes combined providing a set of values showing the mortality experience of a hypothetical group of infants born at the same time and subject throughout their lifetime to the specific mortality rates of a given year (period life tables), from 1950 to 2023. + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/tables/ +tables: + un_wpp_lt: + variables: + central_death_rate: + title: Central death rate + description_short: |- + The death rate, calculated as the number of deaths divided by the average number of people alive during the interval. + description_key: + - |- + The death rate is measured using the number of person-years lived during the interval. + - |- + Person-years refers to the combined total time that a group of people has lived. For example, if 10 people each live for 2 years, they collectively contribute 20 person-years. + - |- + The death rate is slightly different from the 'probability of death' during the interval, because the 'probability of death' metric uses a different denominator: the number of people alive at that age at the start of the interval, while this indicator uses the average number of people alive during the interval. + unit: deaths per 1,000 people + processing_level: minor + description_processing: |- + The original metric is given as a fraction between 0 and 1 (i.e. per-capita). We multiply this by 1,000 to get a per-1,000 people rate. + probability_of_death: + title: Probability of death + unit: "%" + description_short: |- + The probability of dying in a given interval, among people who survived to the start of that interval. + description_key: + - |- + For example, the probability of death for a 50 year old in a given year is found by: dividing the number of deaths in 50 year olds that year, by the number of people alive at the age of 50 at the start of the year. + processing_level: minor + description_processing: |- + The original metric is given as a fraction between 0 and 1 (i.e. per-capita). We multiply this by 100 to get a percentage. + probability_of_survival: + title: Probability of survival + unit: "%" + description_short: The probability that a person who survived until the start of a given interval will have died by the end of the interval. + processing_level: minor + description_processing: |- + The original metric is given as a fraction between 0 and 1 (i.e. per-capita). We multiply this by 100 to get a percentage. + average_survival_length: + title: Average survival length + short_unit: years + unit: years + description_short: Average length of survival between ages x and x+n for persons dying in the interval. + number_survivors: + title: Number of survivors + unit: survivors + description_short: Number of survivors at exact age x, assuming survivors at 0 years old is 100,000. + number_deaths: + title: Number of deaths + short_unit: deaths + unit: deaths + description_short: Number of deaths between ages x and x+n. + number_person_years_lived: + title: Number of person-years lived + unit: person-years + description_short: Number of person-years lived between ages x and x+n. + survivorship_ratio: + title: Survivorship ratio + unit: "" + description_short: Survival ratio (nSx) corresponding to proportion of the life table population in age group (x, x+n) who are alive n year later + number_person_years_remaining: + title: Number of person-years remaining + unit: person-years + description_short: Number of person-years remaining after exact age x. + life_expectancy: + title: Life expectancy + short_unit: years + unit: years + description_short: |- + <%- if type == "period" -%> + The average number of years a person is expected to live, based on mortality rates seen across all age groups in a given interval. + <%- elif type == "cohort" -%> + The average number of years that individuals born in a given interval actually lived, based on tracking that birth cohort over time. + <%- endif -%> + description_key: + - |- + <%- if type == "period" -%> + Life expectancy at age 0 refers to life expectancy at birth. + <%- else -%> + It refers to the remaining life expectancy for people who have already survived to the given age. + <%- endif -%> diff --git a/etl/steps/data/garden/un/2024-12-02/un_wpp_lt.py b/etl/steps/data/garden/un/2024-12-02/un_wpp_lt.py new file mode 100644 index 00000000000..21b1a4d92ba --- /dev/null +++ b/etl/steps/data/garden/un/2024-12-02/un_wpp_lt.py @@ -0,0 +1,100 @@ +"""Load a meadow dataset and create a garden dataset.""" + +import owid.catalog.processing as pr + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) +# Column rename and selection +COLUMNS_RENAME = { + "time": "year", + "agegrpstart": "age", +} +COLUMNS_INDEX = [ + "location", + "year", + "sex", + "age", +] +COLUMNS_INDICATORS = [ + "central_death_rate", + "probability_of_death", + "probability_of_survival", + "number_survivors", + "number_deaths", + "number_person_years_lived", + "survivorship_ratio", + "number_person_years_remaining", + "life_expectancy", + "average_survival_length", +] + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("un_wpp_lt") + + # Read table from meadow dataset. + paths.log.info("load tables, concatenate.") + tb = pr.concat( + [ + ds_meadow["un_wpp_lt_all"].reset_index(), + ds_meadow["un_wpp_lt_f"].reset_index(), + ds_meadow["un_wpp_lt_m"].reset_index(), + ], + short_name=paths.short_name, + ).reset_index() + + # + # Process data. + # + # Sanity check + assert (tb["agegrpspan"].isin([1, -1])).all() and ( + tb.loc[tb["agegrpspan"] == -1, "agegrpstart"] == 100 + ).all(), "Age group span should always be of 1, except for 100+ (-1)" + + # Rename columns, select columns + tb = tb.rename(columns=COLUMNS_RENAME) + + # DTypes + tb = tb.astype( + { + "age": str, + } + ) + + # Change 100 -> 100+ + tb.loc[tb["age"] == "100", "age"] = "100+" + + # Scale central death rates + paths.log.info("scale indicators to make them more.") + tb["central_death_rate"] = tb["central_death_rate"] * 1000 + tb["probability_of_death"] = tb["probability_of_death"] * 100 + tb["probability_of_survival"] = tb["probability_of_survival"] * 100 + + # Harmonize country names. + paths.log.info("harmonise country names.") + tb = geo.harmonize_countries(df=tb, countries_file=paths.country_mapping_path, country_col="location") + + # Harmonize sex sex + tb["sex"] = tb["sex"].map({"Total": "total", "Male": "male", "Female": "female"}) + assert tb["sex"].notna().all(), "NaNs detected after mapping sex values!" + + # Set index + tb = tb.set_index(COLUMNS_INDEX, verify_integrity=True)[COLUMNS_INDICATORS] + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata + ) + + # Save changes in the new garden dataset. + ds_garden.save() diff --git a/etl/steps/data/garden/unep/2023-01-03/renewable_energy_investments.py b/etl/steps/data/garden/unep/2023-01-03/renewable_energy_investments.py index ad1d80f8d94..f3083aacb43 100644 --- a/etl/steps/data/garden/unep/2023-01-03/renewable_energy_investments.py +++ b/etl/steps/data/garden/unep/2023-01-03/renewable_energy_investments.py @@ -1,6 +1,4 @@ -"""Create a garden dataset on renewable energy investments based on UNEP data. - -""" +"""Create a garden dataset on renewable energy investments based on UNEP data.""" from owid import catalog diff --git a/etl/steps/data/garden/unep/2023-12-12/renewable_energy_investments.py b/etl/steps/data/garden/unep/2023-12-12/renewable_energy_investments.py index 49013e474c3..05bfe3ed91d 100644 --- a/etl/steps/data/garden/unep/2023-12-12/renewable_energy_investments.py +++ b/etl/steps/data/garden/unep/2023-12-12/renewable_energy_investments.py @@ -1,6 +1,5 @@ -"""Create a garden dataset on renewable energy investments based on UNEP data. +"""Create a garden dataset on renewable energy investments based on UNEP data.""" -""" from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions. diff --git a/etl/steps/data/garden/unesco/2024-06-16/education_opri.py b/etl/steps/data/garden/unesco/2024-06-16/education_opri.py index d87f1d7bc29..08cd7847bea 100644 --- a/etl/steps/data/garden/unesco/2024-06-16/education_opri.py +++ b/etl/steps/data/garden/unesco/2024-06-16/education_opri.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + from owid.catalog import VariableMeta from etl.data_helpers import geo diff --git a/etl/steps/data/garden/unesco/2024-06-25/education_sdgs.py b/etl/steps/data/garden/unesco/2024-06-25/education_sdgs.py index 9d2e2d8447f..cde00b996f2 100644 --- a/etl/steps/data/garden/unesco/2024-06-25/education_sdgs.py +++ b/etl/steps/data/garden/unesco/2024-06-25/education_sdgs.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + from owid.catalog import VariableMeta from tqdm import tqdm diff --git a/etl/steps/data/garden/unu_wider/2023-11-01/government_revenue_dataset.py b/etl/steps/data/garden/unu_wider/2023-11-01/government_revenue_dataset.py index acc32e890b5..3d281c14dde 100644 --- a/etl/steps/data/garden/unu_wider/2023-11-01/government_revenue_dataset.py +++ b/etl/steps/data/garden/unu_wider/2023-11-01/government_revenue_dataset.py @@ -1,7 +1,15 @@ -"""Load a meadow dataset and create a garden dataset.""" +""" +Load a meadow dataset and create a garden dataset. + +NOTE: To extract the log of the process (to review sanity checks, for example), follow these steps: + 1. Define LONG_FORMAT as True. + 2. Run the following command in the terminal: + nohup uv run etl run government_revenue_dataset > government_revenue_dataset.log 2>&1 & +""" from owid.catalog import Table from structlog import get_logger +from tabulate import tabulate from etl.data_helpers import geo from etl.helpers import PathFinder, create_dataset @@ -11,6 +19,12 @@ # Get paths and naming conventions for current step. paths = PathFinder(__file__) +# Set table format when printing +TABLEFMT = "pretty" + +# Define if I show the full table or just the first 5 rows for assertions +LONG_FORMAT = False + def run(dest_dir: str) -> None: # @@ -27,13 +41,12 @@ def run(dest_dir: str) -> None: # # Process data. - tb = drop_flagged_rows_and_unnecessary_columns(tb) - - # tb = geo.harmonize_countries( df=tb, countries_file=paths.country_mapping_path, ) + tb = drop_flagged_rows_and_unnecessary_columns(tb) + tb = tb.set_index(["country", "year"], verify_integrity=True) # @@ -92,11 +105,74 @@ def drop_flagged_rows_and_unnecessary_columns(tb: Table) -> Table: "cautionnotes", "resourcerevenuenotes", "socialcontributionsnotes", + ] + ) + + tb = sanity_checks(tb) + + # Remove all caution columns + tb = tb.drop( + columns=[ + "caution1accuracyqualityorco", "caution2resourcerevenuestax", "caution3unexcludedresourcere", "caution4inconsistencieswiths", ] - + caution_variables ) return tb + + +def sanity_checks(tb: Table) -> None: + """ + Perform sanity checks on the data. + """ + + tb = tb.copy() + + tb = check_negative_values(tb) + + return tb + + +def check_negative_values(tb: Table): + """ + Check if there are negative values in the variables + """ + + tb = tb.copy() + + # Define columns as all the columns minus country and year + variables = [ + col + for col in tb.columns + if col + not in ["country", "year"] + + [ + "caution1accuracyqualityorco", + "caution2resourcerevenuestax", + "caution3unexcludedresourcere", + "caution4inconsistencieswiths", + ] + ] + + for v in variables: + # Create a mask to check if any value is negative + mask = tb[v] < 0 + any_error = mask.any() + + if any_error: + tb_error = tb[mask].reset_index(drop=True).copy() + paths.log.warning( + f"""{len(tb_error)} observations for {v} are negative: + {_tabulate(tb_error[['country', 'year', 'caution1accuracyqualityorco', 'caution2resourcerevenuestax','caution3unexcludedresourcere','caution4inconsistencieswiths',v]], long_format=LONG_FORMAT)}""" + ) + + return tb + + +def _tabulate(tb: Table, long_format: bool, headers="keys", tablefmt=TABLEFMT, **kwargs): + if long_format: + return tabulate(tb, headers=headers, tablefmt=tablefmt, **kwargs) + else: + return tabulate(tb.head(5), headers=headers, tablefmt=tablefmt, **kwargs) diff --git a/etl/steps/data/garden/urbanization/2024-10-14/ghsl_degree_of_urbanisation.meta.yml b/etl/steps/data/garden/urbanization/2024-10-14/ghsl_degree_of_urbanisation.meta.yml index b12d8afc8f2..7d7c77d2f91 100644 --- a/etl/steps/data/garden/urbanization/2024-10-14/ghsl_degree_of_urbanisation.meta.yml +++ b/etl/steps/data/garden/urbanization/2024-10-14/ghsl_degree_of_urbanisation.meta.yml @@ -9,13 +9,13 @@ definitions: - |- **The Degree of Urbanisation (DEGURBA)** is a method for capturing the urban-rural divide, designed for international comparisons. Developed by six organizations and endorsed by the UN, it uses a two-level classification. - The first level divides areas into cities, towns and semi-dense areas, and rural areas, distinguishing between urban (cities, towns, suburbs) and rural regions. The second level adds detail, splitting towns and rural areas further. + The first level divides areas into cities, towns, and villages, distinguishing between urban (cities, towns, suburbs) and rural regions. The second level adds detail, splitting towns and villages further. This classification is based on 1 km² grid cells, grouped into urban centers, urban clusters, and rural cells. These grids are then used to classify smaller areas, typically using residential population grids from censuses or registers. If detailed data isn't available, a disaggregation grid estimates population distribution. To predict future urbanization (2025 and 2030), both static (land features) and dynamic (past satellite images) components are used to project growth. DEGURBA defines cities by population, not administrative borders, aligning with UN guidelines, though fixed thresholds may not always capture local differences. - description_short: The European Commission combines satellite imagery with national census data to identify [cities](#dod:cities-degurba), [towns and semi-dense areas](#dod:towns-suburbs-degurba), and [rural areas](#dod:rural-areas-degurba) and estimate their respective populations. + description_short: The European Commission combines satellite imagery with national census data to identify [cities](#dod:cities-degurba), [towns](#dod:towns-degurba), and [villages](#dod:villages-degurba) and estimate their respective populations. # Learn more about the available fields: @@ -30,13 +30,13 @@ tables: value: title: |- <% if location_type == "rural_total" and attribute == 'area' and type == 'estimates' %> - Land covered by rural areas + Land covered by villages <% elif location_type == "rural_total" and attribute == 'population' and type == 'estimates' %> - Population living in rural areas + Population living in villages <% elif location_type == "rural_total" and attribute == 'share' and type == 'estimates' %> - Share of land covered by rural areas + Share of land covered by villages <% elif location_type == "rural_total" and attribute == 'popshare' and type == 'estimates' %> - Share of population living in rural areas + Share of population living in villages <% elif location_type == "urban_centre" and attribute == 'area' and type == 'estimates' %> Land covered by cities @@ -48,13 +48,13 @@ tables: Share of population living in cities <% elif location_type == "urban_cluster" and attribute == 'area' and type == 'estimates' %> - Land covered by towns and semi-dense areas + Land covered by towns <% elif location_type == "urban_cluster" and attribute == 'population' and type == 'estimates' %> - Population living in towns and semi-dense areas + Population living in towns <% elif location_type == "urban_cluster" and attribute == 'share' and type == 'estimates' %> - Share of land covered by towns and semi-dense areas + Share of land covered by towns <% elif location_type == "urban_cluster" and attribute == 'popshare' and type == 'estimates' %> - Share of population living in towns and semi-dense areas + Share of population living in towns <% elif location_type == "urban_total" and attribute == 'area' and type == 'estimates' %> Land covered by urban areas @@ -66,13 +66,13 @@ tables: Share of population living in urban areas <% elif location_type == "rural_total" and attribute == 'area' and type == 'projections' %> - Projected land covered by rural areas + Projected land covered by villages <% elif location_type == "rural_total" and attribute == 'population' and type == 'projections' %> - Projected population living in rural areas + Projected population living in villages <% elif location_type == "rural_total" and attribute == 'share' and type == 'projections' %> - Projected share of land covered by rural areas + Projected share of land covered by villages <% elif location_type == "rural_total" and attribute == 'popshare' and type == 'projections' %> - Projected share of population living in rural areas + Projected share of population living in villages <% elif location_type == "urban_centre" and attribute == 'area' and type == 'projections' %> Projected land covered by cities @@ -84,13 +84,13 @@ tables: Projected share of population living in cities <% elif location_type == "urban_cluster" and attribute == 'area' and type == 'projections' %> - Projected land covered by towns and semi-dense areas + Projected land covered by towns <% elif location_type == "urban_cluster" and attribute == 'population' and type == 'projections' %> - Projected population living in towns and semi-dense areas + Projected population living in towns <% elif location_type == "urban_cluster" and attribute == 'share' and type == 'projections' %> - Projected share of land covered by towns and semi-dense areas + Projected share of land covered by towns <% elif location_type == "urban_cluster" and attribute == 'popshare' and type == 'projections' %> - Projected share of population living in towns and semi-dense areas + Projected share of population living in towns <% elif location_type == "urban_total" and attribute == 'area' and type == 'projections' %> Projected land covered by urban areas @@ -107,7 +107,7 @@ tables: <% elif location_type == "semi_dense" and attribute == 'number' and type == 'estimates' %> Number of semi-dense areas <% elif location_type == "rural_total" and attribute == 'number' and type == 'estimates' %> - Number of rural areas + Number of villages <% elif location_type == "urban_centre" and attribute == 'number' and type == 'estimates' %> Number of cities @@ -117,7 +117,7 @@ tables: <% elif location_type == "semi_dense" and attribute == 'number' and type == 'projections' %> Projected number of semi-dense areas <% elif location_type == "rural_total" and attribute == 'number' and type == 'projections' %> - Projected number of rural areas + Projected number of villages <% elif location_type == "urban_centre" and attribute == 'number' and type == 'projections' %> Projected number of cities @@ -127,14 +127,14 @@ tables: Projected population density in cities <% elif location_type == "urban_cluster" and attribute == 'density' and type == 'estimates' %> - Population density in towns and semi-dense areas + Population density in towns <% elif location_type == "urban_cluster" and attribute == 'density' and type == 'projections' %> - Projected population density in towns and semi-dense areas + Projected population density in towns <% elif location_type == "rural_total" and attribute == 'density' and type == 'estimates' %> - Population density in rural areas + Population density in villages <% elif location_type == "rural_total" and attribute == 'density' and type == 'projections' %> - Projected population density in rural areas + Projected population density in villages <% endif %> unit: @@ -160,7 +160,7 @@ tables: <%- endif -%> description_processing: <% if attribute == 'share' or attribute == 'popshare' %> - The share of total area or population for each urbanization level was calculated by dividing the area or population of each level (cities, towns and semi-dense areas, rural areas) by the overall total, providing a percentage representation for each category. + The share of total area or population for each urbanization level was calculated by dividing the area or population of each level (cities, towns, villages) by the overall total, providing a percentage representation for each category. <% elif attribute == 'density' %> Population density was calculated by dividing the population of cities by the total area it covers, providing a measure of the number of people living in each km². <%- endif -%> diff --git a/etl/steps/data/garden/urbanization/2024-12-02/ghsl_urban_centers.countries.json b/etl/steps/data/garden/urbanization/2024-12-02/ghsl_urban_centers.countries.json new file mode 100644 index 00000000000..bdb892107bf --- /dev/null +++ b/etl/steps/data/garden/urbanization/2024-12-02/ghsl_urban_centers.countries.json @@ -0,0 +1,185 @@ +{ + "Afghanistan": "Afghanistan", + "Albania": "Albania", + "Algeria": "Algeria", + "Angola": "Angola", + "Argentina": "Argentina", + "Armenia": "Armenia", + "Aruba": "Aruba", + "Australia": "Australia", + "Austria": "Austria", + "Azerbaijan": "Azerbaijan", + "Bahamas": "Bahamas", + "Bahrain": "Bahrain", + "Bangladesh": "Bangladesh", + "Barbados": "Barbados", + "Belarus": "Belarus", + "Belgium": "Belgium", + "Benin": "Benin", + "Bolivia": "Bolivia", + "Bosnia and Herzegovina": "Bosnia and Herzegovina", + "Botswana": "Botswana", + "Brazil": "Brazil", + "Brunei": "Brunei", + "Bulgaria": "Bulgaria", + "Burkina Faso": "Burkina Faso", + "Burundi": "Burundi", + "Cambodia": "Cambodia", + "Cameroon": "Cameroon", + "Canada": "Canada", + "Central African Republic": "Central African Republic", + "Chad": "Chad", + "Chile": "Chile", + "China": "China", + "Colombia": "Colombia", + "Comoros": "Comoros", + "Costa Rica": "Costa Rica", + "Croatia": "Croatia", + "Cuba": "Cuba", + "Cura\u00e7ao": "Curacao", + "Cyprus": "Cyprus", + "Czechia": "Czechia", + "C\u00f4te d'Ivoire": "Cote d'Ivoire", + "Democratic Republic of the Congo": "Democratic Republic of Congo", + "Denmark": "Denmark", + "Djibouti": "Djibouti", + "Dominican Republic": "Dominican Republic", + "Ecuador": "Ecuador", + "Egypt": "Egypt", + "El Salvador": "El Salvador", + "Equatorial Guinea": "Equatorial Guinea", + "Eritrea": "Eritrea", + "Estonia": "Estonia", + "Ethiopia": "Ethiopia", + "Fiji": "Fiji", + "Finland": "Finland", + "France": "France", + "French Guiana": "French Guiana", + "French Polynesia": "French Polynesia", + "Gabon": "Gabon", + "Georgia": "Georgia", + "Germany": "Germany", + "Ghana": "Ghana", + "Greece": "Greece", + "Guatemala": "Guatemala", + "Guinea-Bissau": "Guinea-Bissau", + "Guyana": "Guyana", + "Haiti": "Haiti", + "Honduras": "Honduras", + "Hungary": "Hungary", + "Iceland": "Iceland", + "India": "India", + "Indonesia": "Indonesia", + "Iran": "Iran", + "Iraq": "Iraq", + "Ireland": "Ireland", + "Israel": "Israel", + "Italy": "Italy", + "Jamaica": "Jamaica", + "Japan": "Japan", + "Jersey": "Jersey", + "Jordan": "Jordan", + "Kazakhstan": "Kazakhstan", + "Kenya": "Kenya", + "Kosovo": "Kosovo", + "Kuwait": "Kuwait", + "Kyrgyzstan": "Kyrgyzstan", + "Laos": "Laos", + "Latvia": "Latvia", + "Lebanon": "Lebanon", + "Lesotho": "Lesotho", + "Liberia": "Liberia", + "Libya": "Libya", + "Lithuania": "Lithuania", + "Luxembourg": "Luxembourg", + "Madagascar": "Madagascar", + "Malawi": "Malawi", + "Malaysia": "Malaysia", + "Maldives": "Maldives", + "Mali": "Mali", + "Malta": "Malta", + "Mauritania": "Mauritania", + "Mauritius": "Mauritius", + "Mayotte": "Mayotte", + "Moldova": "Moldova", + "Mongolia": "Mongolia", + "Montenegro": "Montenegro", + "Morocco": "Morocco", + "Mozambique": "Mozambique", + "Myanmar": "Myanmar", + "Namibia": "Namibia", + "Nepal": "Nepal", + "Netherlands": "Netherlands", + "New Caledonia": "New Caledonia", + "New Zealand": "New Zealand", + "Nicaragua": "Nicaragua", + "Niger": "Niger", + "Nigeria": "Nigeria", + "North Korea": "North Korea", + "North Macedonia": "North Macedonia", + "Northern Cyprus": "Northern Cyprus", + "Norway": "Norway", + "Oman": "Oman", + "Pakistan": "Pakistan", + "Palestine": "Palestine", + "Panama": "Panama", + "Papua New Guinea": "Papua New Guinea", + "Paraguay": "Paraguay", + "Peru": "Peru", + "Philippines": "Philippines", + "Poland": "Poland", + "Portugal": "Portugal", + "Puerto Rico": "Puerto Rico", + "Qatar": "Qatar", + "Romania": "Romania", + "Russia": "Russia", + "Rwanda": "Rwanda", + "R\u00e9union": "Reunion", + "Samoa": "Samoa", + "Saudi Arabia": "Saudi Arabia", + "Senegal": "Senegal", + "Serbia": "Serbia", + "Sierra Leone": "Sierra Leone", + "Singapore": "Singapore", + "Slovakia": "Slovakia", + "Slovenia": "Slovenia", + "Solomon Islands": "Solomon Islands", + "Somalia": "Somalia", + "South Africa": "South Africa", + "South Korea": "South Korea", + "South Sudan": "South Sudan", + "Spain": "Spain", + "Sri Lanka": "Sri Lanka", + "Sudan": "Sudan", + "Suriname": "Suriname", + "Sweden": "Sweden", + "Switzerland": "Switzerland", + "Syria": "Syria", + "Taiwan": "Taiwan", + "Tajikistan": "Tajikistan", + "Tanzania": "Tanzania", + "Thailand": "Thailand", + "Timor-Leste": "East Timor", + "Togo": "Togo", + "Tonga": "Tonga", + "Trinidad and Tobago": "Trinidad and Tobago", + "Tunisia": "Tunisia", + "Turkey": "Turkey", + "Turkmenistan": "Turkmenistan", + "Uganda": "Uganda", + "Ukraine": "Ukraine", + "United Arab Emirates": "United Arab Emirates", + "United Kingdom": "United Kingdom", + "United States": "United States", + "Uruguay": "Uruguay", + "Uzbekistan": "Uzbekistan", + "Vanuatu": "Vanuatu", + "Venezuela": "Venezuela", + "Vietnam": "Vietnam", + "Yemen": "Yemen", + "Zambia": "Zambia", + "Zimbabwe": "Zimbabwe", + "M\u00e9xico": "Mexico", + "Republic of the Congo": "Congo", + "S\u00e3o Tom\u00e9 and Pr\u00edncipe": "Sao Tome and Principe" +} \ No newline at end of file diff --git a/etl/steps/data/garden/urbanization/2024-12-02/ghsl_urban_centers.meta.yml b/etl/steps/data/garden/urbanization/2024-12-02/ghsl_urban_centers.meta.yml new file mode 100644 index 00000000000..a8d7bc66382 --- /dev/null +++ b/etl/steps/data/garden/urbanization/2024-12-02/ghsl_urban_centers.meta.yml @@ -0,0 +1,290 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Urbanization + display: + numDecimalPlaces: + 0 + + processing_level: minor + description_key: + - |- + The European Commission integrates satellite imagery with national census data to delineate the boundaries of capital cities and estimate their populations. + To predict future urbanization (2025 and 2030), both static (land features) and dynamic (past satellite images) components are used to project growth. DEGURBA defines cities by population, not administrative borders, aligning with UN guidelines, though fixed thresholds may not always capture local differences. + + desc_short_density_capital: &desc_short_density_capital |- + The number of people per km² of land area for the country's capital city. + desc_short_density_top_100: &desc_short_density_top_100 |- + The number of people per km² of land area for cities ranked among the top 100 most populous in 2020. + + desc_short_pop_capital: &desc_short_pop_capital |- + The total number of people living in the country's capital city. + desc_short_pop_top_100: &desc_short_pop_top_100 |- + The total number of people living in cities ranked among the top 100 most populous in 2020. + desc_short_pop_1mil: &desc_short_pop_1mil |- + The total number of people living in cities with more than 1 million inhabitants. + + desc_short_1m_total : &desc_short_1m_total |- + The percentage of the total population living in cities with more than 1 million inhabitants. + desc_short_1m_urb: &desc_short_1m_urb |- + The percentage of the urban population living in cities with more than 1 million inhabitants. + + desc_processing_density: &desc_processing_density |- + Population density was calculated by dividing the population of the city by the total area it covers, providing a measure of the number of people living in each km². + + entityAnnotationsMapCapitals: &entityAnnotationsMapCapitals |- + Afghanistan: Kabul + Albania: Tirana + Algeria: Algiers + Angola: Luanda + Argentina: Buenos Aires + Armenia: Yerevan + Aruba: Oranjestad + Australia: North Canberra [Canberra] + Austria: Vienna + Azerbaijan: Baku + Bahamas: Nassau + Bahrain: Manama + Bangladesh: Dhaka + Barbados: Bridgetown + Belarus: Minsk + Belgium: Brussels + Benin: Porto-Novo + Bolivia: La Paz + Bosnia and Herzegovina: Sarajevo + Botswana: Gaborone + Brazil: Brasilia + Brunei: Bandar Seri Begawan + Bulgaria: Sofia + Burkina Faso: Ouagadougou + Burundi: Gitega + Cambodia: Phnom Penh + Cameroon: Yaoundé + Canada: Ottawa + Central African Republic: Bangui + Chad: N'Djamena + Chile: Santiago + China: Beijing + Colombia: Bogota + Comoros: Moroni + Costa Rica: San José + Croatia: Zagreb + Cuba: Havana + Curacao: Willemstad + Cyprus: Strovolos [Nicosia] + Czechia: Prague + Cote d'Ivoire: Yamoussoukro + Democratic Republic of Congo: Kinshasa + Denmark: Copenhagen + Djibouti: Djibouti + Dominican Republic: Santo Domingo + Ecuador: Quito + Egypt: Cairo + El Salvador: San Salvador + Equatorial Guinea: Malabo + Eritrea: Asmara + Estonia: Tallinn + Ethiopia: Addis Ababa + Fiji: Suva + Finland: Helsinki + France: Paris + French Guiana: Cayenne + French Polynesia: Papeete + Gabon: Libreville + Georgia: Tbilisi + Germany: Berlin + Ghana: Accra + Greece: Athens + Guatemala: Guatemala City + Guinea-Bissau: Bissau + Guyana: Georgetown + Haiti: Port-au-Prince + Honduras: Tegucigalpa + Hungary: Budapest + Iceland: Reykjavik + India: New Delhi + Indonesia: Jakarta + Iran: Tehran + Iraq: Baghdad + Ireland: Dublin + Israel: Jerusalem + Italy: Rome + Jamaica: Kingston + Japan: Tokyo + Jersey: St. Helier + Jordan: Amman + Kazakhstan: Astana + Kenya: Nairobi + Kosovo: Pristina + Kuwait: Kuwait City + Kyrgyzstan: Bishkek + Laos: Vientiane + Latvia: Riga + Lebanon: Beirut + Lesotho: Maseru + Liberia: Monrovia + Libya: Tripoli + Lithuania: Vilnius + Luxembourg: Luxembourg + Madagascar: Antananarivo + Malawi: Lilongwe + Malaysia: Kuala Lumpur + Maldives: Malé + Mali: Bamako + Malta: Valletta + Mauritania: Nouakchott + Mauritius: Port Louis + Mayotte: Mamoudzou + Moldova: Chișinău + Mongolia: Ulaanbaatar + Montenegro: Podgorica + Morocco: Rabat + Mozambique: Maputo + Myanmar: Pyinmana [Nay Pyi Taw] + Mexico: Mexico City + Namibia: Windhoek + Nepal: Kathmandu + Netherlands: Amsterdam + New Caledonia: Nouméa + New Zealand: Wellington + Nicaragua: Managua + Niger: Niamey + Nigeria: Abuja + North Korea: P'yŏngyang + North Macedonia: Skopje + Northern Cyprus: Nicosia + Norway: Oslo + Oman: Muscat + Pakistan: Islamabad + Palestine: Ramallah + Panama: Panama City + Papua New Guinea: Port Moresby + Paraguay: Asuncion + Peru: Lima + Philippines: Manila + Poland: Warsaw + Portugal: Lisbon + Puerto Rico: Bayamón [San Juan] + Qatar: Doha + Congo: Brazzaville + Romania: Bucharest + Russia: Moscow + Rwanda: Kigali + Reunion: Saint-Denis + Samoa: Apia + Saudi Arabia: Riyadh + Senegal: Dakar + Serbia: Belgrade + Sierra Leone: Freetown + Singapore: Singapore + Slovakia: Bratislava + Slovenia: Ljubljana + Solomon Islands: Honiara + Somalia: Mogadishu + South Africa: Cape Town + South Korea: Seoul + South Sudan: Juba + Spain: Madrid + Sri Lanka: Colombo [Sri Jayawardenepura Kotte] + Sudan: Khartoum + Suriname: Paramaribo + Sweden: Stockholm + Switzerland: Bern + Syria: Damascus + Sao Tome and Principe: São Tomé + Taiwan: Taipei + Tajikistan: Dushanbe + Tanzania: Dodoma + Thailand: Bangkok + East Timor: Dili + Togo: Lomé + Tonga: Nuku'alofa + Trinidad and Tobago: Port of Spain + Tunisia: Tunis + Turkey: Ankara + Turkmenistan: Ashgabat + Uganda: Kampala + Ukraine: Kyiv + United Arab Emirates: Abu Dhabi + United Kingdom: London + United States: Washington + Uruguay: Montevideo + Uzbekistan: Tashkent + Vanuatu: Port Vila + Venezuela: Caracas + Vietnam: Hanoi + Yemen: Sana'a + Zambia: Lusaka + Zimbabwe: Harare + + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + update_period_days: 365 + + + +tables: + ghsl_urban_centers: + variables: + + urban_pop_projections: + title: Population of the capital city (projected) + unit: 'people' + description_short: *desc_short_pop_capital + display: + entityAnnotationsMap: *entityAnnotationsMapCapitals + isProjection: true + + urban_density_projections: + title: Population density of the capital city (projected) + unit: 'people/km²' + description_short: *desc_short_density_capital + description_processing: *desc_processing_density + display: + entityAnnotationsMap: *entityAnnotationsMapCapitals + isProjection: true + + urban_pop_estimates: + title: Population of the capital city + unit: 'people' + description_short: *desc_short_pop_capital + display: + entityAnnotationsMap: *entityAnnotationsMapCapitals + + urban_density_estimates: + title: Population density of the capital city + unit: 'people/km²' + description_short: *desc_short_density_capital + description_processing: *desc_processing_density + display: + entityAnnotationsMap: *entityAnnotationsMapCapitals + + urban_density_top_100_projections: + title: Population density of the top 100 most populous cities (projected) + unit: 'people/km²' + description_short: *desc_short_density_top_100 + description_processing: *desc_processing_density + display: + isProjection: true + + urban_density_top_100_estimates: + title: Population density of the top 100 most populous cities + unit: 'people/km²' + description_short: *desc_short_density_top_100 + description_processing: *desc_processing_density + + urban_pop_top_100_estimates: + title: Population of the top 100 most populous cities + unit: 'people' + description_short: *desc_short_pop_top_100 + + urban_pop_top_100_projections: + title: Population of the top 100 most populous cities (projected) + unit: 'people' + description_short: *desc_short_pop_top_100 + display: + isProjection: true diff --git a/etl/steps/data/garden/urbanization/2024-12-02/ghsl_urban_centers.py b/etl/steps/data/garden/urbanization/2024-12-02/ghsl_urban_centers.py new file mode 100644 index 00000000000..13558b0f74c --- /dev/null +++ b/etl/steps/data/garden/urbanization/2024-12-02/ghsl_urban_centers.py @@ -0,0 +1,87 @@ +"""Load a meadow dataset and create a garden dataset.""" + +import owid.catalog.processing as pr + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +START_OF_PROJECTIONS = 2025 + +# Regions for which aggregates will be created. +REGIONS = [ + "North America", + "South America", + "Europe", + "Africa", + "Asia", + "Oceania", + "Low-income countries", + "Upper-middle-income countries", + "Lower-middle-income countries", + "High-income countries", +] + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("ghsl_urban_centers") + # Read table from meadow dataset. + tb = ds_meadow.read("ghsl_urban_centers") + + # Load regions dataset. + ds_regions = paths.load_dataset("regions") + + # Load income groups dataset. + ds_income_groups = paths.load_dataset("income_groups") + + # + # Process data. + # + + tb = geo.harmonize_countries(df=tb, countries_file=paths.country_mapping_path) + + tb = tb.drop(columns=["urban_center_name", "urban_area"]) + + # Add region aggregates. + tb = geo.add_regions_to_table( + tb, + aggregations={"urban_pop": "sum"}, + regions=REGIONS, + ds_regions=ds_regions, + ds_income_groups=ds_income_groups, + min_num_values_per_year=1, + ) + + # Split data into estimates and projections. + past_estimates = tb[tb["year"] < START_OF_PROJECTIONS].copy() + future_projections = tb[tb["year"] >= START_OF_PROJECTIONS - 5].copy() + + # Now, for each column, split it into two (projections and estimates). + for col in ["urban_pop", "urban_density", "urban_density_top_100", "urban_pop_top_100"]: + if col not in ["country", "year"]: + past_estimates[f"{col}_estimates"] = tb.loc[tb["year"] < START_OF_PROJECTIONS, col] + future_projections[f"{col}_projections"] = tb.loc[tb["year"] >= START_OF_PROJECTIONS - 5, col] + past_estimates = past_estimates.drop(columns=[col]) + future_projections = future_projections.drop(columns=[col]) + + # Merge past estimates and future projections + tb = pr.merge(past_estimates, future_projections, on=["country", "year"], how="outer") + + tb = tb.format(["country", "year"]) + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata + ) + + # Save changes in the new garden dataset. + ds_garden.save() diff --git a/etl/steps/data/garden/usgs/2024-07-15/historical_statistics_for_mineral_and_material_commodities.py b/etl/steps/data/garden/usgs/2024-07-15/historical_statistics_for_mineral_and_material_commodities.py index 420826d5a87..b5fe034233a 100644 --- a/etl/steps/data/garden/usgs/2024-07-15/historical_statistics_for_mineral_and_material_commodities.py +++ b/etl/steps/data/garden/usgs/2024-07-15/historical_statistics_for_mineral_and_material_commodities.py @@ -86,10 +86,13 @@ # NOTE: The following could be mapped to ("Clays", "Mine, fuller's earth"). We decided to remove "Clays". ("Fuller's earth", "Total"): None, ("Gallium", "Total"): ("Gallium", "Refinery"), + ("Gemstones", "Total"): ("Gemstones", "Mine"), ("Germanium", "Total"): ("Germanium", "Refinery"), ("Gold", "Total"): ("Gold", "Mine"), ("Graphite", "Total"): ("Graphite", "Mine"), ("Gypsum", "Total"): ("Gypsum", "Mine"), + # In USGS current data, "Hafnium" is not reported, only "Zirconium and Hafnium". + ("Hafnium", "Total"): None, ("Helium", "Total"): ("Helium", "Mine"), ("Indium", "Total"): ("Indium", "Refinery"), # NOTE: Industrial diamond production includes natural and synthetic diamonds. @@ -97,13 +100,18 @@ ("Industrial diamond", "Total"): ("Diamond", "Mine and synthetic, industrial"), ("Industrial garnet", "Total"): ("Garnet", "Mine"), ("Industrial sand and gravel", "Total"): ("Sand and gravel", "Mine, industrial"), + ("Iodine", "Total"): ("Iodine", "Mine"), ("Iron Oxide Pigments", "Total"): None, + ("Iron and Steel Scrap", "Total"): None, ("Iron and Steel Slag", "Total"): None, ("Iron ore", "Total"): ("Iron ore", "Mine, crude ore"), # NOTE: The following could be mapped to ("Clays", "Mine, kaolin"). We decided to remove "Clays". ("Kaolin", "Total"): None, + ("Kyanite", "Total"): None, ("Lead", "Total"): ("Lead", "Mine"), ("Lime", "Total"): ("Lime", "Processing"), + ("Lithium statistics", "Total"): ("Lithium", "Mine"), + ("Lumber", "Total"): None, ("Magnesium compounds", "Total"): ("Magnesium compounds", "Mine"), ("Magnesium metal", "Total"): ("Magnesium metal", "Smelter"), ("Manganese", "Total"): ("Manganese", "Mine"), @@ -114,23 +122,34 @@ # NOTE: The following could be mapped to ("Clays", "Mine, miscellaneous"). We decided to remove "Clays". ("Miscellaneous clay", "Total"): None, ("Molybdenum", "Total"): ("Molybdenum", "Mine"), + ("Natural & Synthetic Rutile", "Total"): None, ("Nickel", "Total"): ("Nickel", "Mine"), ("Niobium", "Total"): ("Niobium", "Mine"), # NOTE: Extracted from "world_mine_production". ("Niobium", "Mine"): ("Niobium", "Mine"), ("Nitrogen (Fixed)-Ammonia", "Total"): ("Nitrogen", "Fixed ammonia"), + ("Other industrial wood products", "Total"): None, + ("Paper and board", "Total"): None, # NOTE: The following could be mapped to ("Peat", "Mine"). We decided to remove "Peat". ("Peat", "Total"): None, ("Perlite", "Total"): ("Perlite", "Mine"), ("Phosphate rock", "Total"): ("Phosphate rock", "Mine"), ("Pig Iron", "Total"): ("Iron", "Smelter, pig iron"), + # In USGS current data, PGM are broken down into palladium and platinum. + ("Platinum-group metals", "Total"): None, + ("Plywood and veneer", "Total"): None, + ("Potash", "Total"): ("Potash", "Mine"), ("Pumice and Pumicite", "Total"): ("Pumice and pumicite", "Mine"), + ("Quartz crystal", "Total"): None, + ("Rare earths", "Total"): ("Rare earths", "Mine"), + ("Rhenium", "Total"): ("Rhenium", "Mine"), ("Salt", "Total"): ("Salt", "Mine"), ("Selenium", "Total"): ("Selenium", "Refinery"), ("Silicon", "Total"): ("Silicon", "Processing"), ("Silicon carbide", "Total"): None, ("Silver", "Total"): ("Silver", "Mine"), ("Soda ash", "Total"): ("Soda ash", "Natural and synthetic"), + ("Sodium sulfate", "Total"): None, ("Steel", "Total"): ("Steel", "Processing, crude"), ("Strontium", "Total"): ("Strontium", "Mine"), ("Sulfur", "Total"): ("Sulfur", "Processing"), @@ -138,6 +157,8 @@ ("Tantalum", "Total"): ("Tantalum", "Mine"), # NOTE: The following could be mapped to ("Tellurium", "Refinery"). However, we decided to discard Tellurium. ("Tellurium", "Total"): None, + ("Thallium", "Total"): None, + ("Thorium", "Total"): None, ("Tin", "Total"): ("Tin", "Mine"), # NOTE: For titanium there is no global data. ("Titanium dioxide", "Total"): None, @@ -147,9 +168,15 @@ # To avoid confusion, remove this total. ("Total clay", "Total"): None, ("Total manufactured abrasives ", "Total"): None, + ("Total forestry", "Total"): None, ("Tungsten", "Total"): ("Tungsten", "Mine"), ("Vanadium", "Total"): ("Vanadium", "Mine"), + ("Vermiculite", "Total"): None, + ("Wollastonite", "Total"): None, + ("Wood panel products", "Total"): None, ("Zinc", "Total"): ("Zinc", "Mine"), + # In USGS current data, "Hafnium" is not reported, only "Zirconium and Hafnium". + ("Zirconium", "Total"): None, } # Units can either be "metric tonnes" or "metric tonnes of gross weight". @@ -199,6 +226,8 @@ # NOTE: Bismuth is in gross weight for the US, but metal content for the World. # However, data for the US contains only nans and zeros, and will be removed later on. "Bismuth", + "Gemstones", + "Iodine", ] # Footnotes (that will appear in the footer of charts) to add to the flattened tables (production and unit value). @@ -213,6 +242,8 @@ # "production|Chromium|Mine|tonnes": "Values are reported as tonnes of contained chromium.", "production|Cobalt|Refinery|tonnes": "Values are reported as tonnes of cobalt content.", "production|Bismuth|Mine|tonnes": "Values are reported as tonnes of metal content.", + "production|Lithium|Mine|tonnes": "Values are reported as tonnes of lithium content.", + "production|Gemstones|Mine|tonnes": "Values are reported as tonnes of gemstone-quality diamonds.", } FOOTNOTES_UNIT_VALUE = { "unit_value|Silicon|Processing|constant 1998 US$ per tonne": "Values refer to constant 1998 US$ per tonne of silicon content in ferrosilicon or silicon metal.", @@ -325,9 +356,10 @@ def prepare_us_production(tb: Table, tb_metadata: Table) -> Table: # For now, we'll only keep "production". tb_us_production = tb[["commodity", "year", "production", "unit"]].assign(**{"country": "United States"}) # Remove spurious footnotes like "W". + # Also, zirconiummineral_concentrates contains rows of "< 100,000". It's unclear which value should be assigned here, so we will remove these (two) rows. tb_us_production["production"] = map_series( tb_us_production["production"], - mapping={"W": None}, + mapping={"W": None, "< 100,000": None}, warn_on_missing_mappings=False, warn_on_unused_mappings=True, ).astype({"production": "float64[pyarrow]"}) @@ -640,6 +672,40 @@ def run(dest_dir: str) -> None: (tb_flat["country"].isin(["World"])) & (tb_flat["year"].isin([1974, 1999])), "production|Magnesium metal|Smelter|tonnes", ] = None + + # Lithium production in the US is only given until 1954. From then on, the data is "W" (which means withheld to avoid disclosing company proprietary data). To avoid confusion, simply remove all this data. + error = "Expected lithium US production data to end in 1954. Remove this part of the code." + assert ( + tb_flat.loc[ + (tb_flat["country"] == "United States") & (tb_flat["production|Lithium|Mine|tonnes"].notnull()), "year" + ].max() + == 1954 + ), error + tb_flat.loc[(tb_flat["country"] == "United States"), "production|Lithium|Mine|tonnes"] = None + # Similarly, unit value from 1900 to 1951 is only informed in 1936 (and in terms of production, it's only informed in terms of gross weight, not lithium content). This creates a significant decline in unit value in line charts (between 1936 and 1952) which is unclear if it's real or not. To avoid confusion, ignore that data point and start the series in 1952. + error = ( + "Expected lithium unit value data to only be informed in 1936 (prior to 1952). Remove this part of the code." + ) + assert tb_flat[ + (tb_flat["unit_value|Lithium|Mine|constant 1998 US$ per tonne"].notnull()) & (tb_flat["year"] < 1952) + ]["year"].tolist() == [1936], error + tb_flat.loc[(tb_flat["year"] < 1952), "unit_value|Lithium|Mine|constant 1998 US$ per tonne"] = None + + # Gemstones unit values is zero in a range of years. + # The documentation says "Unit value data for 1922–28 were estimated by interpolation of imports value data, and rounded to two significant figures". + # In practice, the unit values for those years are exactly zero, which are probably spurious. + # Remove those zeros. + _years_with_zero_value = [1922, 1923, 1924, 1925, 1926, 1927, 1928] + error = "Expected gemstones unit value to be zero in a range of years. Remove this part of the code." + assert set( + tb_flat.loc[ + (tb_flat["year"].isin(_years_with_zero_value)) & (tb_flat["country"] == "World"), + "unit_value|Gemstones|Mine|constant 1998 US$ per tonne", + ] + ) == {0.0}, error + tb_flat.loc[ + (tb_flat["year"].isin(_years_with_zero_value)), "unit_value|Gemstones|Mine|constant 1998 US$ per tonne" + ] = None #################################################################################################################### # Format tables conveniently. diff --git a/etl/steps/data/garden/usgs/2024-07-15/mineral_commodity_summaries.py b/etl/steps/data/garden/usgs/2024-07-15/mineral_commodity_summaries.py index 75a4dbf560e..fb6f70e036c 100644 --- a/etl/steps/data/garden/usgs/2024-07-15/mineral_commodity_summaries.py +++ b/etl/steps/data/garden/usgs/2024-07-15/mineral_commodity_summaries.py @@ -3,6 +3,7 @@ All these things are done in a single script because the processes are intertwined. """ + import tempfile import xml.etree.ElementTree as ET from pathlib import Path @@ -1136,9 +1137,9 @@ def run(dest_dir: str) -> None: # Asbestos mine production in Kazakhstan 2020, data says "27400", but in BGS, it is "227400", which is much more # reasonable, looking at prior and posterior data. So it looks like an error in the data. Remove that point. - tb_flat.loc[ - (tb_flat["country"] == "Kazakhstan") & (tb_flat["year"] == 2020), "production|Asbestos|Mine|tonnes" - ] = None + tb_flat.loc[(tb_flat["country"] == "Kazakhstan") & (tb_flat["year"] == 2020), "production|Asbestos|Mine|tonnes"] = ( + None + ) #################################################################################################################### # Format tables conveniently. diff --git a/etl/steps/data/garden/war/2023-07-21/prio_v31.py b/etl/steps/data/garden/war/2023-07-21/prio_v31.py index b211892dade..d5e38c60aee 100644 --- a/etl/steps/data/garden/war/2023-07-21/prio_v31.py +++ b/etl/steps/data/garden/war/2023-07-21/prio_v31.py @@ -21,6 +21,7 @@ 5 = Americas (GWNo: 2-199). - Data for incompatibilities in Oceania are included in region Asia (source decision). Therefore, we have changed the region's name from "Asia" to "Asia and Oceania". """ + from typing import cast import numpy as np diff --git a/etl/steps/data/garden/war/2023-09-21/brecke.meta.yml b/etl/steps/data/garden/war/2023-09-21/brecke.meta.yml index c386e8d7420..dc18bda67ad 100644 --- a/etl/steps/data/garden/war/2023-09-21/brecke.meta.yml +++ b/etl/steps/data/garden/war/2023-09-21/brecke.meta.yml @@ -34,6 +34,7 @@ definitions: number_deaths: description_short: |- + <% set per_capita = "" %> <%- if conflict_type == "all" -%> The estimated number of deaths<< per_capita >> in ongoing armed conflicts that year. diff --git a/etl/steps/data/garden/war/2023-09-21/cow.meta.yml b/etl/steps/data/garden/war/2023-09-21/cow.meta.yml index c0831948820..4fe167ba54f 100644 --- a/etl/steps/data/garden/war/2023-09-21/cow.meta.yml +++ b/etl/steps/data/garden/war/2023-09-21/cow.meta.yml @@ -84,6 +84,7 @@ definitions: number_deaths: description_short: |- + <% set per_capita = "" %> <%- if conflict_type == "all" -%> Included are deaths of combatants due to fighting, disease, and starvation in wars that were ongoing that year<< per_capita >>. diff --git a/etl/steps/data/garden/war/2023-09-21/cow_mid.py b/etl/steps/data/garden/war/2023-09-21/cow_mid.py index 7609d2f5ad2..8497c9a0cf4 100644 --- a/etl/steps/data/garden/war/2023-09-21/cow_mid.py +++ b/etl/steps/data/garden/war/2023-09-21/cow_mid.py @@ -36,7 +36,6 @@ Asia and Oceania: 700-999 """ - import numpy as np import owid.catalog.processing as pr import pandas as pd diff --git a/etl/steps/data/garden/war/2023-09-21/mars.meta.yml b/etl/steps/data/garden/war/2023-09-21/mars.meta.yml index 952866fb2af..56b433fa3ee 100644 --- a/etl/steps/data/garden/war/2023-09-21/mars.meta.yml +++ b/etl/steps/data/garden/war/2023-09-21/mars.meta.yml @@ -41,6 +41,7 @@ definitions: number_deaths: description_short: |- + <% set per_capita = "" %> <%- if conflict_type == "all" -%> The << estimate >> estimate of the number of deaths<< per_capita >> in all ongoing conventional wars that year. diff --git a/etl/steps/data/garden/war/2023-09-21/mie.meta.yml b/etl/steps/data/garden/war/2023-09-21/mie.meta.yml index 597067acb05..99355425c04 100644 --- a/etl/steps/data/garden/war/2023-09-21/mie.meta.yml +++ b/etl/steps/data/garden/war/2023-09-21/mie.meta.yml @@ -13,19 +13,19 @@ definitions: force_usage: &force_usage |- A threat of force can be unspecified, or entail threatening to declare war or to occupy territory; a display of force can entail shows of force or border violations; a use of force can entail attacks, clashes, or battles. conflict_type: - <%- if hostility == "all" -%> + <%- if hostility_level == "all" -%> Included are conflicts between states where force was threatened, displayed, used, or escalated to a war that year - <%- elif hostility == "Threat to use force" -%> + <%- elif hostility_level == "Threat to use force" -%> Included are conflicts between states where, at most, use of force was threatened that year - <%- elif hostility == "Display of force" -%> + <%- elif hostility_level == "Display of force" -%> Included are conflicts between states where, at most, force was displayed that year - <%- elif hostility == "Use of force" -%> + <%- elif hostility_level == "Use of force" -%> Included are conflicts between states where, at most, force was used that year, but there were fewer than 1,000 combatant deaths - <%- elif hostility == "War" -%> + <%- elif hostility_level == "War" -%> Included are interstate wars that were ongoing that year <%- endif -%> @@ -41,10 +41,10 @@ definitions: number_new_conflicts: description_short: |- - <%- if hostility == "Use of force" -%> + <%- if hostility_level == "Use of force" -%> Included are conflicts between states where, at most, force was used that year for the first time, but there were fewer than 1,000 combatant deaths. - <%- elif hostility == "War" -%> + <%- elif hostility_level == "War" -%> Included are interstate wars that started that year. <% else %> @@ -60,6 +60,7 @@ definitions: number_deaths: description_short: |- + <% set per_capita = "" %> The << estimate >> estimate of the number of deaths of combatants due to fighting between states that year<< per_capita >>. description_short_per_capita: <% set per_capita = ", per 100,000 people" %> diff --git a/etl/steps/data/garden/war/2023-09-21/prio_v31.meta.yml b/etl/steps/data/garden/war/2023-09-21/prio_v31.meta.yml index b434cdfee05..0659590c690 100644 --- a/etl/steps/data/garden/war/2023-09-21/prio_v31.meta.yml +++ b/etl/steps/data/garden/war/2023-09-21/prio_v31.meta.yml @@ -173,6 +173,7 @@ tables: unit: deaths description_short: |- <% set estimate = "high" %> + <% set per_capita = "" %> {definitions.number_deaths.description_short} description_key: *description_key_deaths @@ -181,6 +182,7 @@ tables: unit: deaths description_short: |- <% set estimate = "low" %> + <% set per_capita = "" %> {definitions.number_deaths.description_short} description_key: *description_key_deaths @@ -189,6 +191,7 @@ tables: unit: deaths description_short: |- <% set estimate = "best" %> + <% set per_capita = "" %> {definitions.number_deaths.description_short} description_key: *description_key_deaths @@ -197,6 +200,7 @@ tables: unit: deaths per 100,000 people description_short: |- <% set estimate = "high" %> + <% set per_capita = ", per 100,000 people" %> {definitions.number_deaths.description_short} description_key: *description_key_deaths display: @@ -207,6 +211,7 @@ tables: unit: deaths per 100,000 people description_short: |- <% set estimate = "low" %> + <% set per_capita = ", per 100,000 people" %> {definitions.number_deaths.description_short} description_key: *description_key_deaths display: @@ -217,6 +222,7 @@ tables: unit: deaths per 100,000 people description_short: |- <% set estimate = "best" %> + <% set per_capita = ", per 100,000 people" %> {definitions.number_deaths.description_short} description_key: *description_key_deaths @@ -227,6 +233,7 @@ tables: title: Number of ongoing conflicts unit: conflicts description_short: |- + <% set per_capita = "" %> {definitions.number_ongoing_conflicts.description_short}. description_key: *description_key_ongoing @@ -234,6 +241,7 @@ tables: title: Number of ongoing conflicts per state unit: conflicts per state description_short: |- + <% set per_capita = "" %> The number of conflicts divided by the number of all states. This accounts for the changing number of states over time. {definitions.number_ongoing_conflicts.description_short} description_key: *description_key_ongoing display: @@ -243,6 +251,7 @@ tables: title: Number of ongoing conflicts per state-pair unit: conflicts per state-pair description_short: |- + <% set per_capita = "" %> The number of conflicts divided by the number of all state-pairs. This accounts for the changing number of states over time. {definitions.number_ongoing_conflicts.description_short} description_key: *description_key_ongoing display: @@ -255,6 +264,7 @@ tables: title: Number of new conflicts unit: conflicts description_short: |- + <% set per_capita = "" %> {definitions.number_new_conflicts.description_short} description_key: *description_key_new @@ -262,6 +272,7 @@ tables: title: Number of new conflicts per state unit: conflicts per state description_short: |- + <% set per_capita = "" %> The number of conflicts divided by the number of all states. This accounts for the changing number of states over time. {definitions.number_new_conflicts.description_short} description_key: *description_key_new display: @@ -271,6 +282,7 @@ tables: title: Number of new conflicts per state-pair unit: conflicts per state-pair description_short: |- + <% set per_capita = "" %> The number of conflicts divided by the number of all state-pairs. This accounts for the changing number of states over time. {definitions.number_new_conflicts.description_short} description_key: *description_key_new display: diff --git a/etl/steps/data/garden/war/2023-09-21/prio_v31.py b/etl/steps/data/garden/war/2023-09-21/prio_v31.py index 48c28fed0b1..f75d034eb6e 100644 --- a/etl/steps/data/garden/war/2023-09-21/prio_v31.py +++ b/etl/steps/data/garden/war/2023-09-21/prio_v31.py @@ -22,6 +22,7 @@ 5 = Americas (GWNo: 2-199) - The source includes data for incompatibilities in Oceania in region Asia. Therefore, we have changed the region's name from "Asia" to "Asia and Oceania". """ + import numpy as np import owid.catalog.processing as pr import pandas as pd diff --git a/etl/steps/data/garden/war/2023-09-21/ucdp.meta.yml b/etl/steps/data/garden/war/2023-09-21/ucdp.meta.yml index fa0e4ede96d..26fc92220f4 100644 --- a/etl/steps/data/garden/war/2023-09-21/ucdp.meta.yml +++ b/etl/steps/data/garden/war/2023-09-21/ucdp.meta.yml @@ -49,6 +49,7 @@ definitions: # Fields used for number of deaths indicators number_deaths: description_short: |- + <% set per_capita = "" %> <%- if conflict_type == "all" -%> The << estimate >> estimate of the number of deaths of combatants and civilians due to fighting in interstate, intrastate, extrasystemic, non-state conflicts, and one-sided violence that were ongoing that year<< per_capita >>. diff --git a/etl/steps/data/garden/war/2023-09-21/ucdp_prio.meta.yml b/etl/steps/data/garden/war/2023-09-21/ucdp_prio.meta.yml index f1686d8e68f..34960bea396 100644 --- a/etl/steps/data/garden/war/2023-09-21/ucdp_prio.meta.yml +++ b/etl/steps/data/garden/war/2023-09-21/ucdp_prio.meta.yml @@ -2,6 +2,7 @@ definitions: all: description_short: |- + <% set per_capita = "" %> <%- if conflict_type == "all" -%> The << estimate >> estimate of the number of deaths of combatants and civilians due to fighting in interstate, intrastate, extrasystemic, non-state conflicts, and one-sided violence that were ongoing that year<< per_capita >>. diff --git a/etl/steps/data/garden/war/2024-08-26/ucdp.meta.yml b/etl/steps/data/garden/war/2024-08-26/ucdp.meta.yml index 5a54ed362c9..f5f8701bbaf 100644 --- a/etl/steps/data/garden/war/2024-08-26/ucdp.meta.yml +++ b/etl/steps/data/garden/war/2024-08-26/ucdp.meta.yml @@ -49,6 +49,7 @@ definitions: # Fields used for number of deaths indicators number_deaths: description_short: |- + <% set per_capita = "" %> <%- if conflict_type == "all" -%> The << estimate >> estimate of the number of deaths of combatants and civilians due to fighting in interstate, intrastate, extrasystemic, non-state conflicts, and one-sided violence that were ongoing that year<< per_capita >>. @@ -78,6 +79,7 @@ definitions: number_deaths_type: description_short: |- + <% set per_capita = "" %> <%- if conflict_type == "all" -%> The best estimate of the number of deaths of << people_type >> in interstate, intrastate, extrasystemic, non-state conflicts, and one-sided violence that were ongoing that year<< per_capita >>. @@ -91,7 +93,7 @@ definitions: The best estimate of the number of deaths of << people_type >> in non-internationalized intrastate conflicts that were ongoing that year<< per_capita >>. <%- elif conflict_type == "one-sided violence" -%> - The << estimate >> estimate of the number of deaths of << people_type >> from one-sided violence that was ongoing that year<< per_capita >>. + The << estimate | default('') >> estimate of the number of deaths of << people_type >> from one-sided violence that was ongoing that year<< per_capita >>. <%- elif conflict_type == "non-state conflict" -%> The best estimate of the number of deaths of << people_type >> in non-state conflicts that were ongoing that year<< per_capita >>. diff --git a/etl/steps/data/garden/war/2024-08-26/ucdp_prio.meta.yml b/etl/steps/data/garden/war/2024-08-26/ucdp_prio.meta.yml index 33abbd66846..640ad6560c8 100644 --- a/etl/steps/data/garden/war/2024-08-26/ucdp_prio.meta.yml +++ b/etl/steps/data/garden/war/2024-08-26/ucdp_prio.meta.yml @@ -2,6 +2,7 @@ definitions: all: description_short: |- + <% set per_capita = "" %> <%- if conflict_type == "all" -%> The << estimate >> estimate of the number of deaths of combatants and civilians due to fighting in interstate, intrastate, extrasystemic, non-state conflicts, and one-sided violence that were ongoing that year<< per_capita >>. diff --git a/etl/steps/data/garden/war/2024-10-02/ucdp_monthly.meta.yml b/etl/steps/data/garden/war/2024-10-02/ucdp_monthly.meta.yml index da1b751d200..568e6fd686d 100644 --- a/etl/steps/data/garden/war/2024-10-02/ucdp_monthly.meta.yml +++ b/etl/steps/data/garden/war/2024-10-02/ucdp_monthly.meta.yml @@ -49,6 +49,7 @@ definitions: # Fields used for number of deaths indicators number_deaths: description_short: |- + <% set per_capita = "" %> <%- if conflict_type == "all" -%> The << estimate >> estimate of the number of deaths of combatants and civilians due to fighting in interstate, intrastate, extrasystemic, non-state conflicts, and one-sided violence that were ongoing that year<< per_capita >>. @@ -78,6 +79,7 @@ definitions: number_deaths_type: description_short: |- + <% set per_capita = "" %> <%- if conflict_type == "all" -%> The best estimate of the number of deaths of << people_type >> in interstate, intrastate, extrasystemic, non-state conflicts, and one-sided violence that were ongoing that year<< per_capita >>. diff --git a/etl/steps/data/garden/war/2024-11-22/ucdp_preview.meta.yml b/etl/steps/data/garden/war/2024-11-22/ucdp_preview.meta.yml index 024f22d4d27..cc976a6b225 100644 --- a/etl/steps/data/garden/war/2024-11-22/ucdp_preview.meta.yml +++ b/etl/steps/data/garden/war/2024-11-22/ucdp_preview.meta.yml @@ -49,6 +49,7 @@ definitions: # Fields used for number of deaths indicators number_deaths: description_short: |- + <% set per_capita = "" %> <%- if conflict_type == "all" -%> The << estimate >> estimate of the number of deaths of combatants and civilians due to fighting in interstate, intrastate, extrasystemic, non-state conflicts, and one-sided violence that were ongoing that year<< per_capita >>. @@ -78,6 +79,7 @@ definitions: number_deaths_type: description_short: |- + <% set per_capita = "" %> <%- if conflict_type == "all" -%> The best estimate of the number of deaths of << people_type >> in interstate, intrastate, extrasystemic, non-state conflicts, and one-sided violence that were ongoing that year<< per_capita >>. @@ -91,7 +93,7 @@ definitions: The best estimate of the number of deaths of << people_type >> in non-internationalized intrastate conflicts that were ongoing that year<< per_capita >>. <%- elif conflict_type == "one-sided violence" -%> - The << estimate >> estimate of the number of deaths of << people_type >> from one-sided violence that was ongoing that year<< per_capita >>. + The << estimate | default('') >> estimate of the number of deaths of << people_type >> from one-sided violence that was ongoing that year<< per_capita >>. <%- elif conflict_type == "non-state conflict" -%> The best estimate of the number of deaths of << people_type >> in non-state conflicts that were ongoing that year<< per_capita >>. diff --git a/etl/steps/data/garden/wash/2024-01-06/who.py b/etl/steps/data/garden/wash/2024-01-06/who.py index 69911d31c01..ba593e87773 100644 --- a/etl/steps/data/garden/wash/2024-01-06/who.py +++ b/etl/steps/data/garden/wash/2024-01-06/who.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - from owid.catalog import Table from owid.catalog import processing as pr from structlog import get_logger diff --git a/etl/steps/data/garden/wb/2022-10-29/wb_gender/__init__.py b/etl/steps/data/garden/wb/2022-10-29/wb_gender/__init__.py index 7d9a8c53fad..d59e92173f2 100644 --- a/etl/steps/data/garden/wb/2022-10-29/wb_gender/__init__.py +++ b/etl/steps/data/garden/wb/2022-10-29/wb_gender/__init__.py @@ -1,4 +1,5 @@ """WB Gender Garden step.""" + from pathlib import Path from typing import Union diff --git a/etl/steps/data/garden/wb/2023-11-21/worldwide_bureaucracy_indicators.py b/etl/steps/data/garden/wb/2023-11-21/worldwide_bureaucracy_indicators.py index 4b911ba3fb2..0a3b6e6f056 100644 --- a/etl/steps/data/garden/wb/2023-11-21/worldwide_bureaucracy_indicators.py +++ b/etl/steps/data/garden/wb/2023-11-21/worldwide_bureaucracy_indicators.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - from owid.catalog import Table from etl.data_helpers import geo diff --git a/etl/steps/data/garden/wb/2024-06-10/gender_statistics.py b/etl/steps/data/garden/wb/2024-06-10/gender_statistics.py index fe734d701c1..79c4a92e643 100644 --- a/etl/steps/data/garden/wb/2024-06-10/gender_statistics.py +++ b/etl/steps/data/garden/wb/2024-06-10/gender_statistics.py @@ -71,9 +71,10 @@ def run(dest_dir: str) -> None: countries_file=paths.country_mapping_path, ) # Pivot the dataframe so that each indicator is a separate column - tb = tb.pivot(index=["country", "year"], columns="wb_seriescode", values="value") - tb = tb.reset_index() + tb = tb.pivot(index=["country", "year"], columns="wb_seriescode", values="value").reset_index() + # Slovakia - should be the same as Czechoslovakia in the period 1970-1993 for the indicator SG.LAW.EQRM.WK as confirmed by the source (but not yet updated on their database) + tb.loc[(tb["country"] == "Slovakia") & (tb["year"].between(1970, 1993)), "SG.LAW.EQRM.WK"] = 1 # Add metadata by finding the descriptions and sources using indicator codes tb = add_metadata(tb, metadata_tb) diff --git a/etl/steps/data/garden/wb/2024-10-07/world_bank_pip.meta.yml b/etl/steps/data/garden/wb/2024-10-07/world_bank_pip.meta.yml index a4186d774c3..6a971fa5882 100644 --- a/etl/steps/data/garden/wb/2024-10-07/world_bank_pip.meta.yml +++ b/etl/steps/data/garden/wb/2024-10-07/world_bank_pip.meta.yml @@ -406,6 +406,12 @@ tables: - Bangladesh originUrl: https://ourworldindata.org/poverty + poverty_gap_index_215: + presentation: + title_public: Poverty gap index at $2.15 per day + topic_tags: + - Poverty + gini: presentation: title_public: Gini Coefficient diff --git a/etl/steps/data/garden/wb/2024-12-03/poverty_projections.countries.json b/etl/steps/data/garden/wb/2024-12-03/poverty_projections.countries.json new file mode 100644 index 00000000000..7ca68192813 --- /dev/null +++ b/etl/steps/data/garden/wb/2024-12-03/poverty_projections.countries.json @@ -0,0 +1,10 @@ +{ + "EAP": "East Asia and Pacific (PIP)", + "ECA": "Europe and Central Asia (PIP)", + "LAC": "Latin America and the Caribbean (PIP)", + "MNA": "Middle East and North Africa (PIP)", + "OHI": "Other high income countries (PIP)", + "SAS": "South Asia (PIP)", + "SSA": "Sub-Saharan Africa (PIP)", + "World": "World" +} \ No newline at end of file diff --git a/etl/steps/data/garden/wb/2024-12-03/poverty_projections.meta.yml b/etl/steps/data/garden/wb/2024-12-03/poverty_projections.meta.yml new file mode 100644 index 00000000000..f3f04bbd41d --- /dev/null +++ b/etl/steps/data/garden/wb/2024-12-03/poverty_projections.meta.yml @@ -0,0 +1,108 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + processing_level: minor + display: &common-display + tolerance: 0 + entityAnnotationsMap: |- + Other high income countries (PIP): e.g. US, Western Europe, Australia, Japan, South Korea and Saudi Arabia + presentation: + topic_tags: + - Poverty + + description_key_povertyline: |- + <% if povertyline == "2.15" %> + Extreme poverty here is defined as living below the International Poverty Line of $2.15 per day. + <% elif povertyline == "3.65" %> + A poverty line of $3.65 a day represents definitions of national poverty lines in lower-middle-income countries. + <% elif povertyline == "6.85" %> + A poverty line of $6.85 a day represents definitions of national poverty lines in upper-middle-income countries. + <%- endif -%> + + description_key_ppp: |- + The data is measured in international-$ at 2017 prices – this adjusts for inflation and for differences in the cost of living between countries. + + description_key_income_consumption: |- + Depending on the country and year, the data relates to income measured after taxes and benefits, or to consumption, per capita. "Per capita" means that the income of each household is attributed equally to each member of the household (including children). + + description_key_nonmarket_income: |- + Non-market sources of income, including food grown by subsistence farmers for their own consumption, are taken into account. + + description_key_scenarios: |- + <% if scenario == "Historical estimates" %> + Estimates are based on household surveys or extrapolated up until the year of the data release using GDP growth estimates and forecasts. For more details about the methodology, please refer to the [World Bank PIP documentation](https://datanalytics.worldbank.org/PIP-Methodology/lineupestimates.html#nowcasts). + <% elif scenario == "Current forecast + historical growth projections" %> + This data is a projection of the estimates based on GDP growth projections from the World Bank's Global Economic Prospects and the the Macro Poverty Outlook, together with IMF's World Economic Outlook, in the period 2025-2029. For the period 2030-2050, the data is projected using the average annual historical GDP per capita growth over 2010-2019. + <% elif scenario == "Historical estimates + projections" %> + This data combines data based on household surveys or extrapolated up until the year of the data release using GDP growth estimates and forecasts, with projections based on GDP growth projections from the World Bank's Global Economic Prospects and the the Macro Poverty Outlook, together with IMF's World Economic Outlook, in the period 2025-2029. For the period 2030-2050, the data is projected using the average annual historical GDP per capita growth over 2010-2019. + <% elif scenario == "2% growth projections" %> + This data is a projection of the estimates based on a scenario of 2% average GDP per capita growth, while keeping income inequality constant. + <% elif scenario == "2% growth + Gini reduction 1% projections" %> + This data is a projection of the estimates based on a scenatio of 2% average GDP per capita growth, while reducing income inequality by 1% of the Gini coefficient per year. + <% elif scenario == "2% growth + Gini reduction 2% projections" %> + This data is a projection of the estimates based on a scenatio of 2% average GDP per capita growth, while reducing income inequality by 2% of the Gini coefficient per year. + <% elif scenario == "4% growth projections" %> + This data is a projection of the estimates based on a scenario of 4% average GDP per capita growth, while keeping income inequality constant. + <% elif scenario == "6% growth projections" %> + This data is a projection of the estimates based on a scenario of 6% average GDP per capita growth, while keeping income inequality constant. + <% elif scenario == "8% growth projections" %> + This data is a projection of the estimates based on a scenario of 8% average GDP per capita growth, while keeping income inequality constant. + <%- endif -%> + + isprojection_by_scenario: |- + <% if scenario == "Historical estimates" or scenario == "Historical estimates + projections" %> + false + <% else %> + true + <%- endif -%> + + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + title: Poverty projections by the World Bank + update_period_days: 681 + + +tables: + poverty_projections: + variables: + fgt0: + title: $<> a day - Share of population in poverty (<>) + unit: "%" + short_unit: "%" + description_short: "Percentage of population living in households with an income or consumption per person below $<> a day" + description_key: + - "{definitions.description_key_povertyline}" + - "{definitions.description_key_ppp}" + - "{definitions.description_key_income_consumption}" + - "{definitions.description_key_nonmarket_income}" + - "{definitions.description_key_scenarios}" + presentation: + title_public: Share of population living in poverty + title_variant: $<> a day, <> + display: + name: Share of population living below $<> a day (<>) + numDecimalPlaces: 1 + isProjection: "{definitions.isprojection_by_scenario}" + <<: *common-display + + poorpop: + title: $<> a day - Number of people in poverty (<>) + unit: "people" + short_unit: "" + description_short: "Number of people living in households with an income or consumption per person below $<> a day" + description_key: + - "{definitions.description_key_povertyline}" + - "{definitions.description_key_ppp}" + - "{definitions.description_key_income_consumption}" + - "{definitions.description_key_nonmarket_income}" + - "{definitions.description_key_scenarios}" + presentation: + title_public: Number of people living in poverty + title_variant: $<> a day, <> + display: + name: Number of people living below $<> a day (<>) + numDecimalPlaces: 0 + isProjection: "{definitions.isprojection_by_scenario}" + <<: *common-display diff --git a/etl/steps/data/garden/wb/2024-12-03/poverty_projections.py b/etl/steps/data/garden/wb/2024-12-03/poverty_projections.py new file mode 100644 index 00000000000..ec1f7df0dab --- /dev/null +++ b/etl/steps/data/garden/wb/2024-12-03/poverty_projections.py @@ -0,0 +1,136 @@ +"""Load a meadow dataset and create a garden dataset.""" + +import owid.catalog.processing as pr +from owid.catalog import Table +from owid.datautils.dataframes import map_series + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +# Define latest year without projections +LATEST_YEAR_WITHOUT_PROJECTIONS = 2024 + +# Define tables to be loaded. I am not processing country, because they were created for the aggregations and not to highlight them. +TABLES = ["region", "global"] + +# Define scenarios and new names +SCENARIOS = { + "historical": "Historical estimates", + "current_forecast": "Current forecast + historical growth projections", + "2pct": "2% growth projections", + "2pct_gini1": "2% growth + Gini reduction 1% projections", + "2pct_gini2": "2% growth + Gini reduction 2% projections", + "4pct": "4% growth projections", + "6pct": "6% growth projections", + "8pct": "8% growth projections", +} + +# Define index columns +INDEX_COLUMNS = ["country", "year", "povertyline", "scenario"] + +# Define indicator columns +INDICATOR_COLUMNS = ["fgt0", "poorpop"] + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("poverty_projections") + + # Read tables from meadow dataset. + tables = [ds_meadow.read(table_name) for table_name in TABLES] + + # + # Process data. + # + # Concatenate tables + tb = pr.concat(tables, ignore_index=True) + + # Multiply poorpop by 1_000_000 + tb["poorpop"] = tb["poorpop"] * 1_000_000 + + tb = geo.harmonize_countries( + df=tb, + countries_file=paths.country_mapping_path, + ) + + tb = connect_estimates_with_projections(tb) + + # Rename scenario column + tb["scenario"] = map_series( + series=tb["scenario"], + mapping=SCENARIOS, + ) + + # Recover origins + tb["scenario"] = tb["scenario"].copy_metadata(tb["country"]) + + tb = tb.format(INDEX_COLUMNS, short_name="poverty_projections") + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata + ) + + # Save changes in the new garden dataset. + ds_garden.save() + + +def connect_estimates_with_projections(tb: Table) -> Table: + """ + Connects estimates with projections for visualizations in Grapher. + This is repeating the latest estimate in the historical scenario in the rest of the scenarios. + """ + + tb = tb.copy() + + # Save tb_historical and tb_current_forecast, by filtering scenario in historical and current_forecast + tb_historical = tb[tb["scenario"] == "historical"].copy().reset_index(drop=True) + tb_current_forecast = tb[tb["scenario"] == "current_forecast"].copy().reset_index(drop=True) + + # Make table wider, by using scenario as columns + tb = tb.pivot(index=["country", "year", "povertyline"], columns="scenario", values=INDICATOR_COLUMNS) + + # For year LATEST_YEAR_WITHOUT_PROJECTIONS, fill the rest of the columns with the same value + for indicator in INDICATOR_COLUMNS: + for scenario in SCENARIOS.keys(): + if scenario != "historical": + tb.loc[tb.index.get_level_values("year") == LATEST_YEAR_WITHOUT_PROJECTIONS, (indicator, scenario)] = ( + tb.loc[ + tb.index.get_level_values("year") == LATEST_YEAR_WITHOUT_PROJECTIONS, (indicator, scenario) + ].combine_first( + tb.loc[ + tb.index.get_level_values("year") == LATEST_YEAR_WITHOUT_PROJECTIONS, + (indicator, "historical"), + ] + ) + ) + + # Make table long again, by creating a scenario column + tb = tb.stack(level="scenario", future_stack=True).reset_index() + + # Recover origins + for indicator in INDICATOR_COLUMNS: + tb[indicator] = tb[indicator].copy_metadata(tb["country"]) + + # Combine historical and current_forecast, by concatenating tb_historical and tb_current_forecast + tb_connected = pr.concat([tb_historical, tb_current_forecast], ignore_index=True) + + # Rename scenario column to "Historical + current forecast + historical growth" + tb_connected["scenario"] = "Historical estimates + projections" + + # Keep only the columns in INDEX_COLUMNS and INDICATOR_COLUMNS + tb_connected = tb_connected[INDEX_COLUMNS + INDICATOR_COLUMNS] + + # Concatenate tb and tb_connected + tb = pr.concat([tb, tb_connected], ignore_index=True) + + return tb diff --git a/etl/steps/data/garden/who/2022-09-30/ghe.meta.yml b/etl/steps/data/garden/who/2022-09-30/ghe.meta.yml index ccfd9781552..b16d1b679f1 100644 --- a/etl/steps/data/garden/who/2022-09-30/ghe.meta.yml +++ b/etl/steps/data/garden/who/2022-09-30/ghe.meta.yml @@ -113,7 +113,7 @@ definitions: Estimated number of [DALYs](#dod:dalys) from << cause.lower() >> among {definitions.sex} aged {definitions.age}, per 100,000 people. <%- endif -%> footnote: |- - <%- if age == "Age-standardized" -%>To allow for comparisons between countries and over time, this metric is [age-standardized](#dod:age_standardized).<%- endif -%> + <%- if age_group is defined and age_group == "Age-standardized" -%>To allow for comparisons between countries and over time, this metric is [age-standardized](#dod:age_standardized).<%- endif -%> tables: ghe: variables: diff --git a/etl/steps/data/garden/who/2023-04-03/flu_vaccine_policy.py b/etl/steps/data/garden/who/2023-04-03/flu_vaccine_policy.py index e176b91308c..a4e3124db2e 100644 --- a/etl/steps/data/garden/who/2023-04-03/flu_vaccine_policy.py +++ b/etl/steps/data/garden/who/2023-04-03/flu_vaccine_policy.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + import numpy as np import pandas as pd from owid.catalog import Table diff --git a/etl/steps/data/garden/who/2023-06-29/guinea_worm.py b/etl/steps/data/garden/who/2023-06-29/guinea_worm.py index 309b0dd0b0d..0b221d2a202 100644 --- a/etl/steps/data/garden/who/2023-06-29/guinea_worm.py +++ b/etl/steps/data/garden/who/2023-06-29/guinea_worm.py @@ -130,9 +130,9 @@ def update_time_series_with_latest_information(df: Table) -> Table: assert any(df["year"].isin(years_to_add)) df.loc[(df["country"] == "Angola") & (df["year"] >= 2020), "certification_status"] = "Endemic" df.loc[(df["country"] == "Kenya") & (df["year"] >= 2018), "certification_status"] = "Certified disease free" - df.loc[ - (df["country"] == "Democratic Republic of Congo") & (df["year"] >= 2022), "certification_status" - ] = "Certified disease free" + df.loc[(df["country"] == "Democratic Republic of Congo") & (df["year"] >= 2022), "certification_status"] = ( + "Certified disease free" + ) return df diff --git a/etl/steps/data/garden/who/2024-04-26/avian_influenza_ah5n1.py b/etl/steps/data/garden/who/2024-04-26/avian_influenza_ah5n1.py index 2bf144de1fe..fb5e6f7fb31 100644 --- a/etl/steps/data/garden/who/2024-04-26/avian_influenza_ah5n1.py +++ b/etl/steps/data/garden/who/2024-04-26/avian_influenza_ah5n1.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - import owid.catalog.processing as pr import pandas as pd from owid.catalog import Dataset, Table diff --git a/etl/steps/data/garden/who/2024-07-30/ghe.meta.yml b/etl/steps/data/garden/who/2024-07-30/ghe.meta.yml index 0c75bdc4135..b575ed71b7d 100644 --- a/etl/steps/data/garden/who/2024-07-30/ghe.meta.yml +++ b/etl/steps/data/garden/who/2024-07-30/ghe.meta.yml @@ -11,109 +11,109 @@ definitions: - World processing_level: major age: |- - <%- if age_group == "ALLAges" -%> + <% if age_group == "ALLAges" %> all ages - <%- elif age_group == "age-standardized" -%> + <%- elif age_group == "age-standardized" %> an age-standardized population - <%- elif age_group == "YEARS0-14" -%> + <%- elif age_group == "YEARS0-14" %> 0-14 year olds - <%- elif age_group == "YEARS0-4" -%> + <%- elif age_group == "YEARS0-4" %> 0-4 year olds - <%- elif age_group == "YEARS5-14" -%> + <%- elif age_group == "YEARS5-14" %> 5-14 year olds - <%- elif age_group == "YEARS15-19" -%> + <%- elif age_group == "YEARS15-19" %> 15-19 year olds - <%- elif age_group == "YEARS15-49" -%> + <%- elif age_group == "YEARS15-49" %> 15-49 year olds - <%- elif age_group == "YEARS20-24" -%> + <%- elif age_group == "YEARS20-24" %> 20-24 year olds - <%- elif age_group == "YEARS25-34" -%> + <%- elif age_group == "YEARS25-34" %> 25-34 year olds - <%- elif age_group == "YEARS35-44" -%> + <%- elif age_group == "YEARS35-44" %> 35-44 year olds - <%- elif age_group == "YEARS45-54" -%> + <%- elif age_group == "YEARS45-54" %> 45-54 year olds - <%- elif age_group == "YEARS50-69" -%> + <%- elif age_group == "YEARS50-69" %> 50-69 year olds - <%- elif age_group == "YEARS55-64" -%> + <%- elif age_group == "YEARS55-64" %> 55-64 year olds - <%- elif age_group == "YEARS65-74" -%> + <%- elif age_group == "YEARS65-74" %> 65-74 year olds - <%- elif age_group == "YEARS70+" -%> + <%- elif age_group == "YEARS70+" %> 70+ year olds - <%- elif age_group == "YEARS75-84" -%> + <%- elif age_group == "YEARS75-84" %> 75-84 year olds - <%- elif age_group == "YEARS85PLUS" -%> + <%- elif age_group == "YEARS85PLUS" %> 85+ year olds - <%- endif -%> + <%- endif %> sex: |- - <%- if sex == "Both sexes" %>both sexes<% elif sex == "Male" %>males<% elif sex == "Female" %>females<% endif -%> + <% if sex == "Both sexes" %>both sexes<% elif sex == "Male" %>males<% elif sex == "Female" %>females<% endif %> deaths_title: |- - <%- if age_group == "ALLAges" -%> + <% if age_group == "ALLAges" %> Total deaths from << cause.lower() >> among {definitions.sex} - <%- elif age_group == "Age-standardized" -%> + <%- elif age_group == "Age-standardized" %> Age-standardized deaths from << cause.lower() >> among {definitions.sex} - <%- else -%> + <%- else %> Deaths from << cause.lower() >> among {definitions.sex} aged {definitions.age} - <%- endif -%> + <%- endif %> deaths_description: |- - <%- if age_group == "ALLAges" -%> + <% if age_group == "ALLAges" %> Estimated number of deaths from << cause.lower() >> in {definitions.sex}. - <%- elif age_group == "Age-standardized" -%> + <%- elif age_group == "Age-standardized" %> Estimated number of age-standardized deaths from << cause.lower() >> in {definitions.sex}. - <%- else -%> - Estimated number of deaths from << cause.lower() >> among {definitions.sex} aged {definitions.age}. - <%- endif -%> + <%- else %> + Estimated number of deaths from << cause.lower() >> among {definitions.sex} aged {definitions.age}. + <%- endif %> death_rate_title: |- - <%- if age_group == "ALLAges" -%> + <% if age_group == "ALLAges" %> Death rate from << cause.lower() >> among {definitions.sex} - <%- elif age_group == "Age-standardized" -%> + <%- elif age_group == "Age-standardized" %> Age-standardized death rate from << cause.lower() >> among {definitions.sex} - <%- else -%> + <%- else %> Death rate from << cause.lower() >> among {definitions.sex} aged {definitions.age} - <%- endif -%> + <%- endif %> death_rate_description: |- - <%- if age_group == "ALLAges" -%> + <% if age_group == "ALLAges" %> Estimated number of deaths from << cause.lower() >> in {definitions.sex}, per 100,000 people. - <%- elif age_group == "Age-standardized" -%> + <%- elif age_group == "Age-standardized" %> Estimated number of age-standardized deaths from << cause.lower() >> in {definitions.sex}, per 100,000 people. - <%- else -%> + <%- else %> Estimated number of deaths from << cause.lower() >> among {definitions.sex} aged {definitions.age}, per 100,000 people. - <%- endif -%> + <%- endif %> dalys_title: |- - <%- if age_group == "ALLAges" -%> + <% if age_group == "ALLAges" %> DALYs from << cause.lower() >> among {definitions.sex} - <%- elif age_group == "Age-standardized" -%> + <%- elif age_group == "Age-standardized" %> Age-standardized DALYs from << cause.lower() >> among {definitions.sex} - <%- else -%> + <%- else %> DALYs from << cause.lower() >> among {definitions.sex} aged {definitions.age} - <%- endif -%> + <%- endif %> dalys_description: |- - <%- if age_group == "ALLAges" -%> + <% if age_group == "ALLAges" %> Estimated number of [DALYs](#dod:dalys) from << cause.lower() >> in {definitions.sex}. - <%- elif age_group == "Age-standardized" -%> + <%- elif age_group == "Age-standardized" %> Estimated number of age-standardized [DALYs](#dod:dalys) from << cause.lower() >> in {definitions.sex}. - <%- else -%> + <%- else %> Estimated number of [DALYs](#dod:dalys) from << cause.lower() >> among {definitions.sex} aged {definitions.age}. - <%- endif -%> + <%- endif %> dalys_rate_title: |- - <%- if age_group == "ALLAges" -%> + <% if age_group == "ALLAges" %> DALYs from << cause.lower() >>, among {definitions.sex} per 100,000 people - <%- elif age_group == "Age-standardized" -%> + <%- elif age_group == "Age-standardized" %> Age-standardized DALYs from << cause.lower() >> among {definitions.sex}, per 100,000 people - <%- else -%> + <%- else %> DALYs from << cause.lower() >> among {definitions.sex} aged {definitions.age}, per 100,000 people - <%- endif -%> + <%- endif %> dalys_rate_description: |- - <%- if age_group == "ALLAges" -%> + <% if age_group == "ALLAges" %> Estimated number of [DALYs](#dod:dalys) from << cause.lower() >> in {definitions.sex}, per 100,000 people. - <%- elif age_group == "Age-standardized" -%> + <%- elif age_group == "Age-standardized" %> Estimated number of age-standardized [DALYs](#dod:dalys) from << cause.lower() >> in {definitions.sex}, per 100,000 people. - <%- else -%> + <%- else %> Estimated number of [DALYs](#dod:dalys) from << cause.lower() >> among {definitions.sex} aged {definitions.age}, per 100,000 people. - <%- endif -%> + <%- endif %> footnote: |- - <%- if age == "Age-standardized" -%>To allow for comparisons between countries and over time, this metric is [age-standardized](#dod:age_standardized).<%- endif -%> + <% if age_group is defined and age_group == "Age-standardized" %>To allow for comparisons between countries and over time, this metric is [age-standardized](#dod:age_standardized).<% endif %> tables: ghe: variables: diff --git a/etl/steps/data/garden/who/2024-07-30/ghe.py b/etl/steps/data/garden/who/2024-07-30/ghe.py index 43bb9e89d53..bf1e01984a8 100644 --- a/etl/steps/data/garden/who/2024-07-30/ghe.py +++ b/etl/steps/data/garden/who/2024-07-30/ghe.py @@ -77,7 +77,7 @@ def run(dest_dir: str) -> None: tb = rename_table_for_compatibility(tb) if SUBSET: - required_causes = ["Drug use disorders", "Alcohol use disorders"] + required_causes = ["Drug use disorders", "Alcohol use disorders", "Self-harm"] tb = tb[tb.cause.isin(SUBSET.split(",") + required_causes)] # Load countries regions diff --git a/etl/steps/data/garden/who/2024-09-09/flu_test.py b/etl/steps/data/garden/who/2024-09-09/flu_test.py index 9096a3e7768..1c36bd646c6 100644 --- a/etl/steps/data/garden/who/2024-09-09/flu_test.py +++ b/etl/steps/data/garden/who/2024-09-09/flu_test.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - import numpy as np import pandas as pd from owid.catalog import Table diff --git a/etl/steps/data/garden/who/latest/avian_influenza_ah5n1.py b/etl/steps/data/garden/who/latest/avian_influenza_ah5n1.py index 549e7b14eb1..28f7ac6c271 100644 --- a/etl/steps/data/garden/who/latest/avian_influenza_ah5n1.py +++ b/etl/steps/data/garden/who/latest/avian_influenza_ah5n1.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - import owid.catalog.processing as pr import pandas as pd from owid.catalog import Dataset, Table diff --git a/etl/steps/data/garden/who/latest/fluid.py b/etl/steps/data/garden/who/latest/fluid.py index 0f9d24c5d58..7e0076c2230 100644 --- a/etl/steps/data/garden/who/latest/fluid.py +++ b/etl/steps/data/garden/who/latest/fluid.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - from datetime import datetime import numpy as np diff --git a/etl/steps/data/garden/who/latest/flunet.py b/etl/steps/data/garden/who/latest/flunet.py index 8ee3bca2ea7..56f1a8f7c74 100644 --- a/etl/steps/data/garden/who/latest/flunet.py +++ b/etl/steps/data/garden/who/latest/flunet.py @@ -1,4 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" + import numpy as np import pandas as pd from owid.catalog import Table diff --git a/etl/steps/data/garden/wid/2023-08-24/world_inequality_database.py b/etl/steps/data/garden/wid/2023-08-24/world_inequality_database.py index 1ad59630fa3..71a440fad30 100644 --- a/etl/steps/data/garden/wid/2023-08-24/world_inequality_database.py +++ b/etl/steps/data/garden/wid/2023-08-24/world_inequality_database.py @@ -1,6 +1,5 @@ """Load World Inequality Database meadow dataset and create a garden dataset.""" - import owid.catalog.processing as pr from owid.catalog import Dataset, Table from shared import add_metadata_vars, add_metadata_vars_distribution diff --git a/etl/steps/data/garden/wpf/2024-10-03/famines.py b/etl/steps/data/garden/wpf/2024-10-03/famines.py index d898d1da1db..219e92faab7 100644 --- a/etl/steps/data/garden/wpf/2024-10-03/famines.py +++ b/etl/steps/data/garden/wpf/2024-10-03/famines.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - import pandas as pd from owid.catalog import Dataset, Table diff --git a/etl/steps/data/garden/wpf/2024-10-03/famines_by_place.py b/etl/steps/data/garden/wpf/2024-10-03/famines_by_place.py index b1d23fe069a..f8b77714a8a 100644 --- a/etl/steps/data/garden/wpf/2024-10-03/famines_by_place.py +++ b/etl/steps/data/garden/wpf/2024-10-03/famines_by_place.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - import numpy as np import owid.catalog.processing as pr import pandas as pd diff --git a/etl/steps/data/garden/wpf/2024-10-03/famines_by_regime_gdp.py b/etl/steps/data/garden/wpf/2024-10-03/famines_by_regime_gdp.py index 8b9872e33f5..ff31dc141ae 100644 --- a/etl/steps/data/garden/wpf/2024-10-03/famines_by_regime_gdp.py +++ b/etl/steps/data/garden/wpf/2024-10-03/famines_by_regime_gdp.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - import owid.catalog.processing as pr from owid.catalog import Dataset, Table diff --git a/etl/steps/data/garden/wpf/2024-10-03/total_famines_by_year_decade.py b/etl/steps/data/garden/wpf/2024-10-03/total_famines_by_year_decade.py index 5067cd0df3f..7f2b6df93b1 100644 --- a/etl/steps/data/garden/wpf/2024-10-03/total_famines_by_year_decade.py +++ b/etl/steps/data/garden/wpf/2024-10-03/total_famines_by_year_decade.py @@ -1,6 +1,5 @@ """Load a meadow dataset and create a garden dataset.""" - import numpy as np import owid.catalog.processing as pr import pandas as pd diff --git a/etl/steps/data/grapher/antibiotics/2024-11-12/antimicrobial_usage.py b/etl/steps/data/grapher/antibiotics/2024-11-12/antimicrobial_usage.py index 1750c76a66b..9a24fbc4608 100644 --- a/etl/steps/data/grapher/antibiotics/2024-11-12/antimicrobial_usage.py +++ b/etl/steps/data/grapher/antibiotics/2024-11-12/antimicrobial_usage.py @@ -16,6 +16,7 @@ def run(dest_dir: str) -> None: # Read table from garden dataset. tb_class = ds_garden["class"] tb_aware = ds_garden["aware"] + tb_class_agg = ds_garden["class_aggregated"] # # Process data. @@ -26,7 +27,10 @@ def run(dest_dir: str) -> None: # # Create a new grapher dataset with the same metadata as the garden dataset. ds_grapher = create_dataset( - dest_dir, tables=[tb_class, tb_aware], check_variables_metadata=True, default_metadata=ds_garden.metadata + dest_dir, + tables=[tb_class, tb_aware, tb_class_agg], + check_variables_metadata=True, + default_metadata=ds_garden.metadata, ) # Save changes in the new grapher dataset. diff --git a/etl/steps/data/grapher/antibiotics/2024-11-20/bloodstream_amr.py b/etl/steps/data/grapher/antibiotics/2024-12-02/microbe_amr.py similarity index 65% rename from etl/steps/data/grapher/antibiotics/2024-11-20/bloodstream_amr.py rename to etl/steps/data/grapher/antibiotics/2024-12-02/microbe_amr.py index 74525ddf3d8..f54fcc98772 100644 --- a/etl/steps/data/grapher/antibiotics/2024-11-20/bloodstream_amr.py +++ b/etl/steps/data/grapher/antibiotics/2024-12-02/microbe_amr.py @@ -11,18 +11,17 @@ def run(dest_dir: str) -> None: # Load inputs. # # Load garden dataset. - ds_garden = paths.load_dataset("bloodstream_amr") + ds_garden = paths.load_dataset("microbe_amr") # Read table from garden dataset. - tb = ds_garden.read("bloodstream_amr", reset_index=False) - tb_amr = ds_garden.read("amr_entity", reset_index=False) + tb = ds_garden.read("microbe_amr", reset_index=False) # # Save outputs. # # Create a new grapher dataset with the same metadata as the garden dataset. ds_grapher = create_dataset( - dest_dir, tables=[tb, tb_amr], check_variables_metadata=True, default_metadata=ds_garden.metadata + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_garden.metadata ) # Save changes in the new grapher dataset. diff --git a/etl/steps/data/grapher/antibiotics/2024-12-02/microbe_neonatal_amr.py b/etl/steps/data/grapher/antibiotics/2024-12-02/microbe_neonatal_amr.py new file mode 100644 index 00000000000..885a0c65b02 --- /dev/null +++ b/etl/steps/data/grapher/antibiotics/2024-12-02/microbe_neonatal_amr.py @@ -0,0 +1,28 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("microbe_neonatal_amr") + + # Read table from garden dataset. + tb = ds_garden.read("microbe_neonatal_amr", reset_index=False) + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/antibiotics/2024-12-02/total_pathogen_bloodstream.py b/etl/steps/data/grapher/antibiotics/2024-12-02/total_pathogen_bloodstream.py new file mode 100644 index 00000000000..3738ea0149f --- /dev/null +++ b/etl/steps/data/grapher/antibiotics/2024-12-02/total_pathogen_bloodstream.py @@ -0,0 +1,28 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("total_pathogen_bloodstream") + + # Read table from garden dataset. + tb = ds_garden.read("total_pathogen_bloodstream", reset_index=False) + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.py b/etl/steps/data/grapher/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.py new file mode 100644 index 00000000000..81a269f0ae0 --- /dev/null +++ b/etl/steps/data/grapher/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.py @@ -0,0 +1,28 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("total_pathogen_bloodstream_amr") + + # Read table from garden dataset. + tb = ds_garden.read("total_pathogen_bloodstream_amr", reset_index=False) + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/antibiotics/2024-12-03/glass_enrolment.py b/etl/steps/data/grapher/antibiotics/2024-12-03/glass_enrolment.py new file mode 100644 index 00000000000..3f1604a2896 --- /dev/null +++ b/etl/steps/data/grapher/antibiotics/2024-12-03/glass_enrolment.py @@ -0,0 +1,28 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("glass_enrolment") + + # Read table from garden dataset. + tb = ds_garden.read("glass_enrolment", reset_index=False) + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/antibiotics/2024-12-04/microbe_total_pathogens.py b/etl/steps/data/grapher/antibiotics/2024-12-04/microbe_total_pathogens.py new file mode 100644 index 00000000000..898f3506d2b --- /dev/null +++ b/etl/steps/data/grapher/antibiotics/2024-12-04/microbe_total_pathogens.py @@ -0,0 +1,28 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("microbe_total_pathogens") + + # Read table from garden dataset. + tb = ds_garden.read("microbe_total_pathogens", reset_index=False) + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/antibiotics/2024-12-04/microbe_total_pathogens_amr.py b/etl/steps/data/grapher/antibiotics/2024-12-04/microbe_total_pathogens_amr.py new file mode 100644 index 00000000000..2a5df838f2e --- /dev/null +++ b/etl/steps/data/grapher/antibiotics/2024-12-04/microbe_total_pathogens_amr.py @@ -0,0 +1,28 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("microbe_total_pathogens_amr") + + # Read table from garden dataset. + tb = ds_garden.read("microbe_total_pathogens_amr", reset_index=False) + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/antibiotics/2024-12-05/microbe_neonatal_total_amr.py b/etl/steps/data/grapher/antibiotics/2024-12-05/microbe_neonatal_total_amr.py new file mode 100644 index 00000000000..6a743b902c6 --- /dev/null +++ b/etl/steps/data/grapher/antibiotics/2024-12-05/microbe_neonatal_total_amr.py @@ -0,0 +1,28 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("microbe_neonatal_total_amr") + + # Read table from garden dataset. + tb = ds_garden.read("microbe_neonatal_total_amr", reset_index=False) + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.py b/etl/steps/data/grapher/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.py new file mode 100644 index 00000000000..407f17d98ab --- /dev/null +++ b/etl/steps/data/grapher/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.py @@ -0,0 +1,28 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("microbe_total_deaths_by_syndrome_amr") + + # Read table from garden dataset. + tb = ds_garden.read("microbe_total_deaths_by_syndrome_amr", reset_index=False) + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-06-19/epoch_compute_intensive_organizations.py b/etl/steps/data/grapher/artificial_intelligence/2024-06-19/epoch_compute_intensive_organizations.py index 9478c5e5e42..6f232db8993 100644 --- a/etl/steps/data/grapher/artificial_intelligence/2024-06-19/epoch_compute_intensive_organizations.py +++ b/etl/steps/data/grapher/artificial_intelligence/2024-06-19/epoch_compute_intensive_organizations.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + from etl.helpers import PathFinder, create_dataset, grapher_checks # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-07-10/epoch_aggregates_organizations.py b/etl/steps/data/grapher/artificial_intelligence/2024-07-10/epoch_aggregates_organizations.py index f479f165881..52c7abe49d9 100644 --- a/etl/steps/data/grapher/artificial_intelligence/2024-07-10/epoch_aggregates_organizations.py +++ b/etl/steps/data/grapher/artificial_intelligence/2024-07-10/epoch_aggregates_organizations.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + from etl.helpers import PathFinder, create_dataset, grapher_checks # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-08-05/epoch_aggregates_organizations.py b/etl/steps/data/grapher/artificial_intelligence/2024-08-05/epoch_aggregates_organizations.py index f479f165881..52c7abe49d9 100644 --- a/etl/steps/data/grapher/artificial_intelligence/2024-08-05/epoch_aggregates_organizations.py +++ b/etl/steps/data/grapher/artificial_intelligence/2024-08-05/epoch_aggregates_organizations.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + from etl.helpers import PathFinder, create_dataset, grapher_checks # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-08-05/epoch_compute_intensive_organizations.py b/etl/steps/data/grapher/artificial_intelligence/2024-08-05/epoch_compute_intensive_organizations.py index 9478c5e5e42..6f232db8993 100644 --- a/etl/steps/data/grapher/artificial_intelligence/2024-08-05/epoch_compute_intensive_organizations.py +++ b/etl/steps/data/grapher/artificial_intelligence/2024-08-05/epoch_compute_intensive_organizations.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + from etl.helpers import PathFinder, create_dataset, grapher_checks # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-09-09/epoch_aggregates_organizations.py b/etl/steps/data/grapher/artificial_intelligence/2024-09-09/epoch_aggregates_organizations.py index f479f165881..52c7abe49d9 100644 --- a/etl/steps/data/grapher/artificial_intelligence/2024-09-09/epoch_aggregates_organizations.py +++ b/etl/steps/data/grapher/artificial_intelligence/2024-09-09/epoch_aggregates_organizations.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + from etl.helpers import PathFinder, create_dataset, grapher_checks # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-09-09/epoch_compute_intensive_organizations.py b/etl/steps/data/grapher/artificial_intelligence/2024-09-09/epoch_compute_intensive_organizations.py index 9478c5e5e42..6f232db8993 100644 --- a/etl/steps/data/grapher/artificial_intelligence/2024-09-09/epoch_compute_intensive_organizations.py +++ b/etl/steps/data/grapher/artificial_intelligence/2024-09-09/epoch_compute_intensive_organizations.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + from etl.helpers import PathFinder, create_dataset, grapher_checks # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-10-01/epoch_aggregates_organizations.py b/etl/steps/data/grapher/artificial_intelligence/2024-10-01/epoch_aggregates_organizations.py index f479f165881..52c7abe49d9 100644 --- a/etl/steps/data/grapher/artificial_intelligence/2024-10-01/epoch_aggregates_organizations.py +++ b/etl/steps/data/grapher/artificial_intelligence/2024-10-01/epoch_aggregates_organizations.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + from etl.helpers import PathFinder, create_dataset, grapher_checks # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-10-01/epoch_compute_intensive_organizations.py b/etl/steps/data/grapher/artificial_intelligence/2024-10-01/epoch_compute_intensive_organizations.py index 9478c5e5e42..6f232db8993 100644 --- a/etl/steps/data/grapher/artificial_intelligence/2024-10-01/epoch_compute_intensive_organizations.py +++ b/etl/steps/data/grapher/artificial_intelligence/2024-10-01/epoch_compute_intensive_organizations.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + from etl.helpers import PathFinder, create_dataset, grapher_checks # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch.py b/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch.py index 82ece67e6f9..e8c91465a15 100644 --- a/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch.py +++ b/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch.py @@ -37,13 +37,13 @@ def run(dest_dir: str) -> None: # Update metadata for col in ["max_compute", "max_parameters", "max_data"]: - tb[col].metadata.origins = tb["system"].metadata.origins + tb[col].metadata.origins = tb["model"].metadata.origins # Drop year as we don't need it anymore tb = tb.drop("year", axis=1) # Rename for plotting model name as country in grapher - tb = tb.rename(columns={"system": "country", "days_since_1949": "year"}) + tb = tb.rename(columns={"model": "country", "days_since_1949": "year"}) tb = tb.format(["country", "year"]) # diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch_aggregates_organizations.py b/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch_aggregates_organizations.py index f479f165881..52c7abe49d9 100644 --- a/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch_aggregates_organizations.py +++ b/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch_aggregates_organizations.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + from etl.helpers import PathFinder, create_dataset, grapher_checks # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch_compute_intensive.py b/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch_compute_intensive.py index 323e67bd023..f6df4df4e55 100644 --- a/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch_compute_intensive.py +++ b/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch_compute_intensive.py @@ -20,7 +20,7 @@ def run(dest_dir: str) -> None: # Process data. # # Rename for plotting model name as country in grapher - tb = tb.rename_index_names({"system": "country", "days_since_1949": "year"}) + tb = tb.rename_index_names({"model": "country", "days_since_1949": "year"}) # # Save outputs. # diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch_compute_intensive_organizations.py b/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch_compute_intensive_organizations.py index 9478c5e5e42..6f232db8993 100644 --- a/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch_compute_intensive_organizations.py +++ b/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch_compute_intensive_organizations.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + from etl.helpers import PathFinder, create_dataset, grapher_checks # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch_regressions.py b/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch_regressions.py index e3d4b17b89d..8c21dfbbc5a 100644 --- a/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch_regressions.py +++ b/etl/steps/data/grapher/artificial_intelligence/2024-11-03/epoch_regressions.py @@ -15,7 +15,7 @@ def run(dest_dir: str) -> None: # Read table from garden dataset. tb = ds_garden["epoch_regressions"] - tb = tb.rename_index_names({"system": "country", "days_since_1949": "year"}) + tb = tb.rename_index_names({"model": "country", "days_since_1949": "year"}) # # Save outputs. # diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch.meta.yml b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch.meta.yml new file mode 100644 index 00000000000..af50f790b40 --- /dev/null +++ b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch.meta.yml @@ -0,0 +1,18 @@ +definitions: + common: + unit: '' + short_unit: '' + display: + zeroDay: '1949-01-01' + yearIsDay: true + +tables: + epoch: + variables: + max_compute: + title: Maximum compute + max_data: + title: Maximum data + max_parameters: + title: Maximum parameters + diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch.py b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch.py new file mode 100644 index 00000000000..6db83e94816 --- /dev/null +++ b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch.py @@ -0,0 +1,102 @@ +"""Load a garden dataset and create a grapher dataset.""" + +import owid.catalog.processing as pr +import pandas as pd +from owid.catalog import Table + +from etl.helpers import PathFinder, create_dataset, grapher_checks + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("epoch") + + # Read table from garden dataset. + tb = ds_garden["epoch"].reset_index() + # + # Process data. + # + # Extract year from 'publication_date' and create a new 'year' column + tb["year"] = tb["publication_date"].dt.year + + # For visualization purposes I am adding the rows with the maximum values of compute, data, and parameters in each year to the table as a separate "model". I don't want to do this in garden as it'd affect other datasets that depend on this one. + columns = { + "training_computation_petaflop": "compute", + "training_dataset_size__datapoints": "data", + "parameters": "parameters", + } + # Find maximum values for a given column (compute, data, params) per year, label them, and add summary rows. + for column, label in columns.items(): + tb = find_max_label_and_concat(tb, column, label) + + # Update metadata + for col in ["max_compute", "max_parameters", "max_data"]: + tb[col].metadata.origins = tb["model"].metadata.origins + + # Drop year as we don't need it anymore + tb = tb.drop("year", axis=1) + + # Rename for plotting model name as country in grapher + tb = tb.rename(columns={"model": "country", "days_since_1949": "year"}) + tb = tb.format(["country", "year"]) + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb], default_metadata=ds_garden.metadata, check_variables_metadata=True + ) + + # + # Checks. + # + grapher_checks(ds_grapher) + + # Save changes in the new grapher dataset. + ds_grapher.save() + + +def find_max_label_and_concat(tb, column, label): + """ + Find maximum values for a given column per year, label them, and add summary rows. + + This function: + 1. Identifies rows with maximum values for the specified column in each year. + 2. Labels these maximum value rows in a new column using their original model names. + 3. Creates new summary rows for these maximum values. + 4. Adds these new summary rows to the original table. + + Note: + - Creates a new column named f"max_{label}" to indicate maximum values. + - Preserves original data and model names. + - Adds new summary rows with "model" set to f"Maximum {label}". + """ + tb = tb.sort_values(by=["year"]) # Ensure the DataFrame is sorted by year + max_value = -float("inf") + rows_to_keep = [] + + for _, row in tb.iterrows(): + if not pd.isna(row[column]) and row[column] > max_value: + max_value = row[column] + rows_to_keep.append(row) + + tb_filtered = Table(rows_to_keep) + + idx = tb_filtered[[column, "year"]].fillna(0).groupby("year")[column].idxmax() + + tb_filtered[f"max_{label}"] = "Other" + tb_filtered.loc[idx, f"max_{label}"] = f"Maximum {label}" + + max_rows = tb_filtered.loc[idx].copy() + max_rows["model"] = f"Maximum {label}" + + tb = pr.concat([tb, max_rows], ignore_index=True) + + return tb diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_aggregates_affiliation.py b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_aggregates_affiliation.py new file mode 100644 index 00000000000..6582a86db80 --- /dev/null +++ b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_aggregates_affiliation.py @@ -0,0 +1,41 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset, grapher_checks + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("epoch_aggregates_affiliation") + + # Read table from garden dataset. + tb = ds_garden["epoch_aggregates_affiliation"] + + # + # Process data. + # + # Rename for plotting research affiliation as country in grapher + tb = tb.rename_index_names( + { + "organization_categorization": "country", + } + ) + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset(dest_dir, tables=[tb], default_metadata=ds_garden.metadata) + + # + # Checks. + # + grapher_checks(ds_grapher) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_aggregates_countries.py b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_aggregates_countries.py new file mode 100644 index 00000000000..658d7982804 --- /dev/null +++ b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_aggregates_countries.py @@ -0,0 +1,30 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset, grapher_checks + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("epoch_aggregates_countries") + + # Read table from garden dataset. + tb_garden = ds_garden["epoch_aggregates_countries"] + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset(dest_dir, tables=[tb_garden], default_metadata=ds_garden.metadata) + + # + # Checks. + # + grapher_checks(ds_grapher) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_aggregates_domain.py b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_aggregates_domain.py new file mode 100644 index 00000000000..fb2fa66d43b --- /dev/null +++ b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_aggregates_domain.py @@ -0,0 +1,39 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset, grapher_checks + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("epoch_aggregates_domain") + + # Read table from garden dataset. + tb = ds_garden["epoch_aggregates_domain"] + # + # Process data. + # + # Rename for plotting model domain as country in grapher + tb = tb.rename_index_names( + { + "domain": "country", + } + ) + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset(dest_dir, tables=[tb], default_metadata=ds_garden.metadata) + + # + # Checks. + # + grapher_checks(ds_grapher) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_aggregates_organizations.py b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_aggregates_organizations.py new file mode 100644 index 00000000000..52c7abe49d9 --- /dev/null +++ b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_aggregates_organizations.py @@ -0,0 +1,39 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset, grapher_checks + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("epoch_aggregates_organizations") + + # Read table from garden dataset. + tb = ds_garden["epoch_aggregates_organizations"] + # + # Process data. + # + # Rename for plotting model domain as country in grapher + tb = tb.rename_index_names( + { + "organization": "country", + } + ) + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset(dest_dir, tables=[tb], default_metadata=ds_garden.metadata) + + # + # Checks. + # + grapher_checks(ds_grapher) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_compute_intensive.py b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_compute_intensive.py new file mode 100644 index 00000000000..f6df4df4e55 --- /dev/null +++ b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_compute_intensive.py @@ -0,0 +1,33 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("epoch_compute_intensive") + + # Read table from garden dataset. + tb = ds_garden["epoch_compute_intensive"] + + # + # Process data. + # + # Rename for plotting model name as country in grapher + tb = tb.rename_index_names({"model": "country", "days_since_1949": "year"}) + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_compute_intensive_countries.py b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_compute_intensive_countries.py new file mode 100644 index 00000000000..ef0aea55b10 --- /dev/null +++ b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_compute_intensive_countries.py @@ -0,0 +1,30 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset, grapher_checks + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("epoch_compute_intensive_countries") + + # Read table from garden dataset. + tb_garden = ds_garden["epoch_compute_intensive_countries"] + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset(dest_dir, tables=[tb_garden], default_metadata=ds_garden.metadata) + + # + # Checks. + # + grapher_checks(ds_grapher) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_compute_intensive_domain.py b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_compute_intensive_domain.py new file mode 100644 index 00000000000..efb5fea33ce --- /dev/null +++ b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_compute_intensive_domain.py @@ -0,0 +1,39 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset, grapher_checks + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("epoch_compute_intensive_domain") + + # Read table from garden dataset. + tb = ds_garden["epoch_compute_intensive_domain"] + # + # Process data. + # + # Rename for plotting model domain as country in grapher + tb = tb.rename_index_names( + { + "domain": "country", + } + ) + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset(dest_dir, tables=[tb], default_metadata=ds_garden.metadata) + + # + # Checks. + # + grapher_checks(ds_grapher) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_compute_intensive_organizations.py b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_compute_intensive_organizations.py new file mode 100644 index 00000000000..6f232db8993 --- /dev/null +++ b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_compute_intensive_organizations.py @@ -0,0 +1,39 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset, grapher_checks + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("epoch_compute_intensive_organizations") + + # Read table from garden dataset. + tb = ds_garden["epoch_compute_intensive_organizations"] + # + # Process data. + # + # Rename for plotting model domain as country in grapher + tb = tb.rename_index_names( + { + "organization": "country", + } + ) + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset(dest_dir, tables=[tb], default_metadata=ds_garden.metadata) + + # + # Checks. + # + grapher_checks(ds_grapher) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_regressions.py b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_regressions.py new file mode 100644 index 00000000000..8c21dfbbc5a --- /dev/null +++ b/etl/steps/data/grapher/artificial_intelligence/2024-12-05/epoch_regressions.py @@ -0,0 +1,33 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset, grapher_checks + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("epoch_regressions") + + # Read table from garden dataset. + tb = ds_garden["epoch_regressions"] + tb = tb.rename_index_names({"model": "country", "days_since_1949": "year"}) + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb], default_metadata=ds_garden.metadata, check_variables_metadata=True + ) + + # + # Checks. + # + grapher_checks(ds_grapher) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/biodiversity/2023-01-11/cherry_blossom.py b/etl/steps/data/grapher/biodiversity/2023-01-11/cherry_blossom.py index b9a8ac4d893..434806130bf 100644 --- a/etl/steps/data/grapher/biodiversity/2023-01-11/cherry_blossom.py +++ b/etl/steps/data/grapher/biodiversity/2023-01-11/cherry_blossom.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/biodiversity/2024-01-25/cherry_blossom.py b/etl/steps/data/grapher/biodiversity/2024-01-25/cherry_blossom.py index 146ce79b4a7..dc8556eab7c 100644 --- a/etl/steps/data/grapher/biodiversity/2024-01-25/cherry_blossom.py +++ b/etl/steps/data/grapher/biodiversity/2024-01-25/cherry_blossom.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/climate/2023-12-20/surface_country_level_monthly_anomaly.py b/etl/steps/data/grapher/climate/2023-12-20/surface_country_level_monthly_anomaly.py index 7ad100d786d..75d6fb20996 100644 --- a/etl/steps/data/grapher/climate/2023-12-20/surface_country_level_monthly_anomaly.py +++ b/etl/steps/data/grapher/climate/2023-12-20/surface_country_level_monthly_anomaly.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/climate/2023-12-20/surface_global_monthly_anomaly.py b/etl/steps/data/grapher/climate/2023-12-20/surface_global_monthly_anomaly.py index 151e53cb783..b1f20e4b543 100644 --- a/etl/steps/data/grapher/climate/2023-12-20/surface_global_monthly_anomaly.py +++ b/etl/steps/data/grapher/climate/2023-12-20/surface_global_monthly_anomaly.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/climate/2023-12-20/surface_global_monthly_anomaly_all_countries.py b/etl/steps/data/grapher/climate/2023-12-20/surface_global_monthly_anomaly_all_countries.py index e6d942302cb..0fe7dca5ef9 100644 --- a/etl/steps/data/grapher/climate/2023-12-20/surface_global_monthly_anomaly_all_countries.py +++ b/etl/steps/data/grapher/climate/2023-12-20/surface_global_monthly_anomaly_all_countries.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/climate/2023-12-20/surface_temperature.py b/etl/steps/data/grapher/climate/2023-12-20/surface_temperature.py index d8c8e082922..3e0ed1dc84d 100644 --- a/etl/steps/data/grapher/climate/2023-12-20/surface_temperature.py +++ b/etl/steps/data/grapher/climate/2023-12-20/surface_temperature.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - import pandas as pd from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/climate/2024-01-31/climate_change_impacts_annual.py b/etl/steps/data/grapher/climate/2024-01-31/climate_change_impacts_annual.py index d2ce85e4a2d..9e24b6978c7 100644 --- a/etl/steps/data/grapher/climate/2024-01-31/climate_change_impacts_annual.py +++ b/etl/steps/data/grapher/climate/2024-01-31/climate_change_impacts_annual.py @@ -1,7 +1,4 @@ -"""Load a garden dataset and create a grapher dataset. - -""" - +"""Load a garden dataset and create a grapher dataset.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/climate/2024-01-31/climate_change_impacts_monthly.py b/etl/steps/data/grapher/climate/2024-01-31/climate_change_impacts_monthly.py index c69428bae1b..4538d048ff7 100644 --- a/etl/steps/data/grapher/climate/2024-01-31/climate_change_impacts_monthly.py +++ b/etl/steps/data/grapher/climate/2024-01-31/climate_change_impacts_monthly.py @@ -1,6 +1,4 @@ -"""Load a garden dataset and create a grapher dataset. - -""" +"""Load a garden dataset and create a grapher dataset.""" from etl.grapher_helpers import adapt_table_with_dates_to_grapher from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/climate/2024-03-11/climate_change_impacts_annual.py b/etl/steps/data/grapher/climate/2024-03-11/climate_change_impacts_annual.py index d2ce85e4a2d..9e24b6978c7 100644 --- a/etl/steps/data/grapher/climate/2024-03-11/climate_change_impacts_annual.py +++ b/etl/steps/data/grapher/climate/2024-03-11/climate_change_impacts_annual.py @@ -1,7 +1,4 @@ -"""Load a garden dataset and create a grapher dataset. - -""" - +"""Load a garden dataset and create a grapher dataset.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/climate/2024-03-11/climate_change_impacts_monthly.py b/etl/steps/data/grapher/climate/2024-03-11/climate_change_impacts_monthly.py index c69428bae1b..4538d048ff7 100644 --- a/etl/steps/data/grapher/climate/2024-03-11/climate_change_impacts_monthly.py +++ b/etl/steps/data/grapher/climate/2024-03-11/climate_change_impacts_monthly.py @@ -1,6 +1,4 @@ -"""Load a garden dataset and create a grapher dataset. - -""" +"""Load a garden dataset and create a grapher dataset.""" from etl.grapher_helpers import adapt_table_with_dates_to_grapher from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/climate/2024-04-17/climate_change_impacts_annual.py b/etl/steps/data/grapher/climate/2024-04-17/climate_change_impacts_annual.py index d2ce85e4a2d..9e24b6978c7 100644 --- a/etl/steps/data/grapher/climate/2024-04-17/climate_change_impacts_annual.py +++ b/etl/steps/data/grapher/climate/2024-04-17/climate_change_impacts_annual.py @@ -1,7 +1,4 @@ -"""Load a garden dataset and create a grapher dataset. - -""" - +"""Load a garden dataset and create a grapher dataset.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/climate/2024-04-17/climate_change_impacts_monthly.py b/etl/steps/data/grapher/climate/2024-04-17/climate_change_impacts_monthly.py index c69428bae1b..4538d048ff7 100644 --- a/etl/steps/data/grapher/climate/2024-04-17/climate_change_impacts_monthly.py +++ b/etl/steps/data/grapher/climate/2024-04-17/climate_change_impacts_monthly.py @@ -1,6 +1,4 @@ -"""Load a garden dataset and create a grapher dataset. - -""" +"""Load a garden dataset and create a grapher dataset.""" from etl.grapher_helpers import adapt_table_with_dates_to_grapher from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/climate/2024-05-20/climate_change_impacts_annual.py b/etl/steps/data/grapher/climate/2024-05-20/climate_change_impacts_annual.py index d2ce85e4a2d..9e24b6978c7 100644 --- a/etl/steps/data/grapher/climate/2024-05-20/climate_change_impacts_annual.py +++ b/etl/steps/data/grapher/climate/2024-05-20/climate_change_impacts_annual.py @@ -1,7 +1,4 @@ -"""Load a garden dataset and create a grapher dataset. - -""" - +"""Load a garden dataset and create a grapher dataset.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/climate/2024-05-20/climate_change_impacts_monthly.py b/etl/steps/data/grapher/climate/2024-05-20/climate_change_impacts_monthly.py index c69428bae1b..4538d048ff7 100644 --- a/etl/steps/data/grapher/climate/2024-05-20/climate_change_impacts_monthly.py +++ b/etl/steps/data/grapher/climate/2024-05-20/climate_change_impacts_monthly.py @@ -1,6 +1,4 @@ -"""Load a garden dataset and create a grapher dataset. - -""" +"""Load a garden dataset and create a grapher dataset.""" from etl.grapher_helpers import adapt_table_with_dates_to_grapher from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/climate/2024-07-23/climate_change_impacts_annual.py b/etl/steps/data/grapher/climate/2024-07-23/climate_change_impacts_annual.py index d2ce85e4a2d..9e24b6978c7 100644 --- a/etl/steps/data/grapher/climate/2024-07-23/climate_change_impacts_annual.py +++ b/etl/steps/data/grapher/climate/2024-07-23/climate_change_impacts_annual.py @@ -1,7 +1,4 @@ -"""Load a garden dataset and create a grapher dataset. - -""" - +"""Load a garden dataset and create a grapher dataset.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/climate/2024-07-23/climate_change_impacts_monthly.py b/etl/steps/data/grapher/climate/2024-07-23/climate_change_impacts_monthly.py index c69428bae1b..4538d048ff7 100644 --- a/etl/steps/data/grapher/climate/2024-07-23/climate_change_impacts_monthly.py +++ b/etl/steps/data/grapher/climate/2024-07-23/climate_change_impacts_monthly.py @@ -1,6 +1,4 @@ -"""Load a garden dataset and create a grapher dataset. - -""" +"""Load a garden dataset and create a grapher dataset.""" from etl.grapher_helpers import adapt_table_with_dates_to_grapher from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/climate/2024-07-23/sea_ice_anomalies_by_month.py b/etl/steps/data/grapher/climate/2024-07-23/sea_ice_anomalies_by_month.py index b16879ec60b..424ebf91f27 100644 --- a/etl/steps/data/grapher/climate/2024-07-23/sea_ice_anomalies_by_month.py +++ b/etl/steps/data/grapher/climate/2024-07-23/sea_ice_anomalies_by_month.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from owid.catalog import Table from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/climate/2024-09-30/climate_change_impacts_annual.py b/etl/steps/data/grapher/climate/2024-09-30/climate_change_impacts_annual.py index d2ce85e4a2d..9e24b6978c7 100644 --- a/etl/steps/data/grapher/climate/2024-09-30/climate_change_impacts_annual.py +++ b/etl/steps/data/grapher/climate/2024-09-30/climate_change_impacts_annual.py @@ -1,7 +1,4 @@ -"""Load a garden dataset and create a grapher dataset. - -""" - +"""Load a garden dataset and create a grapher dataset.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/climate/2024-09-30/climate_change_impacts_monthly.py b/etl/steps/data/grapher/climate/2024-09-30/climate_change_impacts_monthly.py index c69428bae1b..4538d048ff7 100644 --- a/etl/steps/data/grapher/climate/2024-09-30/climate_change_impacts_monthly.py +++ b/etl/steps/data/grapher/climate/2024-09-30/climate_change_impacts_monthly.py @@ -1,6 +1,4 @@ -"""Load a garden dataset and create a grapher dataset. - -""" +"""Load a garden dataset and create a grapher dataset.""" from etl.grapher_helpers import adapt_table_with_dates_to_grapher from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/climate/2024-09-30/sea_ice_anomalies_by_month.py b/etl/steps/data/grapher/climate/2024-09-30/sea_ice_anomalies_by_month.py index b16879ec60b..424ebf91f27 100644 --- a/etl/steps/data/grapher/climate/2024-09-30/sea_ice_anomalies_by_month.py +++ b/etl/steps/data/grapher/climate/2024-09-30/sea_ice_anomalies_by_month.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from owid.catalog import Table from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/climate/2024-11-18/climate_change_impacts_annual.py b/etl/steps/data/grapher/climate/2024-11-18/climate_change_impacts_annual.py index d2ce85e4a2d..9e24b6978c7 100644 --- a/etl/steps/data/grapher/climate/2024-11-18/climate_change_impacts_annual.py +++ b/etl/steps/data/grapher/climate/2024-11-18/climate_change_impacts_annual.py @@ -1,7 +1,4 @@ -"""Load a garden dataset and create a grapher dataset. - -""" - +"""Load a garden dataset and create a grapher dataset.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/climate/2024-11-18/climate_change_impacts_monthly.py b/etl/steps/data/grapher/climate/2024-11-18/climate_change_impacts_monthly.py index c69428bae1b..4538d048ff7 100644 --- a/etl/steps/data/grapher/climate/2024-11-18/climate_change_impacts_monthly.py +++ b/etl/steps/data/grapher/climate/2024-11-18/climate_change_impacts_monthly.py @@ -1,6 +1,4 @@ -"""Load a garden dataset and create a grapher dataset. - -""" +"""Load a garden dataset and create a grapher dataset.""" from etl.grapher_helpers import adapt_table_with_dates_to_grapher from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/climate/2024-11-18/sea_ice_anomalies_by_month.py b/etl/steps/data/grapher/climate/2024-11-18/sea_ice_anomalies_by_month.py index 9c1bcfce4d5..93ce4fd0bb3 100644 --- a/etl/steps/data/grapher/climate/2024-11-18/sea_ice_anomalies_by_month.py +++ b/etl/steps/data/grapher/climate/2024-11-18/sea_ice_anomalies_by_month.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from owid.catalog import Table from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/covid/latest/countries_reporting.py b/etl/steps/data/grapher/covid/latest/countries_reporting.py index 8d9d1c11080..6e6aa4f81aa 100644 --- a/etl/steps/data/grapher/covid/latest/countries_reporting.py +++ b/etl/steps/data/grapher/covid/latest/countries_reporting.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/covid/latest/vaccinations_age.py b/etl/steps/data/grapher/covid/latest/vaccinations_age.py index 020030ea484..8a1be405c77 100644 --- a/etl/steps/data/grapher/covid/latest/vaccinations_age.py +++ b/etl/steps/data/grapher/covid/latest/vaccinations_age.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/covid/latest/vaccinations_manufacturer.py b/etl/steps/data/grapher/covid/latest/vaccinations_manufacturer.py index eb588c4a970..40ac012bf9c 100644 --- a/etl/steps/data/grapher/covid/latest/vaccinations_manufacturer.py +++ b/etl/steps/data/grapher/covid/latest/vaccinations_manufacturer.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/covid/latest/vaccinations_us.py b/etl/steps/data/grapher/covid/latest/vaccinations_us.py index dbb0ad8fde2..3c1d0eaee0a 100644 --- a/etl/steps/data/grapher/covid/latest/vaccinations_us.py +++ b/etl/steps/data/grapher/covid/latest/vaccinations_us.py @@ -1,7 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" -from shared import to_grapher_date - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. @@ -21,9 +19,6 @@ def run(dest_dir: str) -> None: # # Process data. # - # Grapher date - tb = to_grapher_date(tb, "2021-01-01") - # Rename state -> country for grapher tb = tb.rename_index_names( { diff --git a/etl/steps/data/grapher/demography/2022-11-30/life_expectancy.py b/etl/steps/data/grapher/demography/2022-11-30/life_expectancy.py index 553b382aeb2..0eb053e6c3b 100644 --- a/etl/steps/data/grapher/demography/2022-11-30/life_expectancy.py +++ b/etl/steps/data/grapher/demography/2022-11-30/life_expectancy.py @@ -1,4 +1,5 @@ """Grapher step for our Life Expectancy OMM.""" + import yaml from owid import catalog diff --git a/etl/steps/data/grapher/demography/2023-03-30/population.py b/etl/steps/data/grapher/demography/2023-03-30/population.py index 60c5b25ce52..64f889d6637 100644 --- a/etl/steps/data/grapher/demography/2023-03-30/population.py +++ b/etl/steps/data/grapher/demography/2023-03-30/population.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + import re from copy import deepcopy from typing import Any, List @@ -184,8 +185,8 @@ def _create_metric_version_from_mask( if title_suffix: table[metric_new].metadata.title = f"{table[metric_new].metadata.title} {title_suffix}" if display_name_suffix: - table[metric_new].metadata.display[ - "name" - ] = f"{table[metric_new].metadata.display['name']} {display_name_suffix}" + table[metric_new].metadata.display["name"] = ( + f"{table[metric_new].metadata.display['name']} {display_name_suffix}" + ) table[metric_new].metadata.description = description return table.astype({metric_new: dtype}) diff --git a/etl/steps/data/grapher/demography/2023-03-31/population.py b/etl/steps/data/grapher/demography/2023-03-31/population.py index f29d8d6ddd4..64f66a99fd2 100644 --- a/etl/steps/data/grapher/demography/2023-03-31/population.py +++ b/etl/steps/data/grapher/demography/2023-03-31/population.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + import re from copy import deepcopy from typing import Any, List diff --git a/etl/steps/data/grapher/demography/2024-01-25/population.py b/etl/steps/data/grapher/demography/2024-01-25/population.py index 32d04c7497e..6c0a0e2701e 100644 --- a/etl/steps/data/grapher/demography/2024-01-25/population.py +++ b/etl/steps/data/grapher/demography/2024-01-25/population.py @@ -2,6 +2,7 @@ This dataset is a work in progress. We have layed down the foundations for the step, but are waiting until new data is available (both from Maddison and from 2024 WPP). """ + import re from copy import deepcopy from typing import Any, List diff --git a/etl/steps/data/grapher/demography/2024-07-15/population.py b/etl/steps/data/grapher/demography/2024-07-15/population.py index 6308675a7d7..a2f9caffbae 100644 --- a/etl/steps/data/grapher/demography/2024-07-15/population.py +++ b/etl/steps/data/grapher/demography/2024-07-15/population.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/demography/2024-11-26/multiple_births.py b/etl/steps/data/grapher/demography/2024-11-26/multiple_births.py new file mode 100644 index 00000000000..4940b624e48 --- /dev/null +++ b/etl/steps/data/grapher/demography/2024-11-26/multiple_births.py @@ -0,0 +1,28 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("multiple_births") + + # Read table from garden dataset. + tb = ds_garden.read("multiple_births", reset_index=False) + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/demography/2024-12-02/survivor_percentiles.py b/etl/steps/data/grapher/demography/2024-12-02/survivor_percentiles.py new file mode 100644 index 00000000000..1e319eaee4c --- /dev/null +++ b/etl/steps/data/grapher/demography/2024-12-02/survivor_percentiles.py @@ -0,0 +1,35 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("survivor_percentiles") + + # Read table from garden dataset. + + # + # Process data. + # + tables = list(ds_garden) + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, + tables=tables, + check_variables_metadata=True, + default_metadata=ds_garden.metadata, + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/demography/2024-12-03/birth_rate.py b/etl/steps/data/grapher/demography/2024-12-03/birth_rate.py new file mode 100644 index 00000000000..dc16db838f9 --- /dev/null +++ b/etl/steps/data/grapher/demography/2024-12-03/birth_rate.py @@ -0,0 +1,28 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("birth_rate") + + # Read table from garden dataset. + tb = ds_garden.read("birth_rate", reset_index=False) + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/demography/2024-12-03/broken_limits_le.py b/etl/steps/data/grapher/demography/2024-12-03/broken_limits_le.py new file mode 100644 index 00000000000..5c223bd8430 --- /dev/null +++ b/etl/steps/data/grapher/demography/2024-12-03/broken_limits_le.py @@ -0,0 +1,32 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("broken_limits_le") + + # Read table from garden dataset. + tb = ds_garden["broken_limits_le"] + + # + # Process data. + # + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/demography/2024-12-03/fertility_rate.py b/etl/steps/data/grapher/demography/2024-12-03/fertility_rate.py new file mode 100644 index 00000000000..fd49f86a998 --- /dev/null +++ b/etl/steps/data/grapher/demography/2024-12-03/fertility_rate.py @@ -0,0 +1,28 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("fertility_rate") + + # Read table from garden dataset. + tb = ds_garden.read("fertility_rate", reset_index=False) + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/demography/2024-12-03/gini_le.py b/etl/steps/data/grapher/demography/2024-12-03/gini_le.py new file mode 100644 index 00000000000..27d2649971e --- /dev/null +++ b/etl/steps/data/grapher/demography/2024-12-03/gini_le.py @@ -0,0 +1,32 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("gini_le") + + # Read table from garden dataset. + tables = list(ds_garden) + + # + # Process data. + # + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=tables, check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/demography/2024-12-03/life_expectancy.py b/etl/steps/data/grapher/demography/2024-12-03/life_expectancy.py new file mode 100644 index 00000000000..1e23557767e --- /dev/null +++ b/etl/steps/data/grapher/demography/2024-12-03/life_expectancy.py @@ -0,0 +1,32 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("life_expectancy") + + # Read table from garden dataset. + tables = list(ds_garden) + + # + # Process data. + # + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=tables, check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/demography/2024-12-03/life_tables.py b/etl/steps/data/grapher/demography/2024-12-03/life_tables.py new file mode 100644 index 00000000000..53efc672ef1 --- /dev/null +++ b/etl/steps/data/grapher/demography/2024-12-03/life_tables.py @@ -0,0 +1,69 @@ +"""There is some work to filter only those indicators and dimensions that are relevant for the grapher. + +That is, we may just want a subset of the indicators, and few single-age groups. +""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) +# Relevant indicators +INDICATORS_RELEVANT = [ + "central_death_rate", + "life_expectancy", + "probability_of_death", +] +INDICATORS_RELEVANT_REL = [ + "life_expectancy_fm_diff", + "life_expectancy_fm_ratio", + "central_death_rate_mf_ratio", +] +# Single-age groups to preserve +AGES_SINGLE = [ + 0, + 10, + 15, + 25, + 45, + 65, + 80, +] +AGES_SINGLE = list(map(str, AGES_SINGLE)) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("life_tables") + + # Read table from garden dataset. + tb = ds_garden.read("life_tables") + tb_diff = ds_garden.read("diff_ratios") + + # + # Process data. + # + ## Only keep particular ages + tb = tb.loc[tb["age"].isin(AGES_SINGLE)] + tb_diff = tb_diff.loc[tb_diff["age"].isin(AGES_SINGLE)] + + ## Set index back + tb = tb.format(["country", "year", "sex", "age", "type"]) + tb_diff = tb_diff.format(["country", "year", "age", "type"]) + + ## Only keep subset of columns + tb = tb[INDICATORS_RELEVANT] + tb_diff = tb_diff[INDICATORS_RELEVANT_REL] + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb, tb_diff], check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/demography/2024-12-03/phi_gender_le.py b/etl/steps/data/grapher/demography/2024-12-03/phi_gender_le.py new file mode 100644 index 00000000000..04c5c3ceccf --- /dev/null +++ b/etl/steps/data/grapher/demography/2024-12-03/phi_gender_le.py @@ -0,0 +1,28 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("phi_gender_le") + + # Read table from garden dataset. + tables = list(ds_garden) + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=tables, check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/demography/2024-12-05/paternal_ages.py b/etl/steps/data/grapher/demography/2024-12-05/paternal_ages.py new file mode 100644 index 00000000000..98ec6766363 --- /dev/null +++ b/etl/steps/data/grapher/demography/2024-12-05/paternal_ages.py @@ -0,0 +1,28 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("paternal_ages") + + # Read table from garden dataset. + tb = ds_garden.read("paternal_age", reset_index=False) + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/demography/2024-12-06/wittgenstein_human_capital.py b/etl/steps/data/grapher/demography/2024-12-06/wittgenstein_human_capital.py new file mode 100644 index 00000000000..c739d9b9a63 --- /dev/null +++ b/etl/steps/data/grapher/demography/2024-12-06/wittgenstein_human_capital.py @@ -0,0 +1,48 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +AGE_ACCEPTED = [ + "total", + "15+", + "15-19", + "20-39", + "40-64", + "65+", +] + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("wittgenstein_human_capital") + + # Read table from garden dataset. + tables = {t.m.short_name: t for t in list(ds_garden)} + + # Filter out some dimensions, to make the step faster + tb = tables["by_sex_age_edu"] + index_cols = tb.index.names + tb = tb.reset_index() + tb = tb.loc[(tb["sex"] == "total") & (tb["age"].isin(AGE_ACCEPTED))] + tables["by_sex_age_edu"] = tb.format(index_cols) + + # + # Save outputs. + # + tables = list(tables.values()) + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, + tables=tables, + check_variables_metadata=True, + default_metadata=ds_garden.metadata, + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/demography/2024-12-06/wittgenstein_human_capital_proj.py b/etl/steps/data/grapher/demography/2024-12-06/wittgenstein_human_capital_proj.py new file mode 100644 index 00000000000..0210be8e5dc --- /dev/null +++ b/etl/steps/data/grapher/demography/2024-12-06/wittgenstein_human_capital_proj.py @@ -0,0 +1,48 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +AGE_ACCEPTED = [ + "total", + "15+", + "15-19", + "20-39", + "40-64", + "65+", +] + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("wittgenstein_human_capital_proj") + + # Read table from garden dataset. + tables = {t.m.short_name: t for t in list(ds_garden)} + + # Filter out some dimensions, to make the step faster + tb = tables["by_sex_age_edu"] + index_cols = tb.index.names + tb = tb.reset_index() + tb = tb.loc[(tb["sex"] == "total") & (tb["age"].isin(AGE_ACCEPTED))] + tables["by_sex_age_edu"] = tb.format(index_cols) + + # + # Save outputs. + # + tables = list(tables.values()) + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, + tables=tables, + check_variables_metadata=True, + default_metadata=ds_garden.metadata, + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/education/2024-12-11/people_with_education.py b/etl/steps/data/grapher/education/2024-12-11/people_with_education.py new file mode 100644 index 00000000000..7de59114b85 --- /dev/null +++ b/etl/steps/data/grapher/education/2024-12-11/people_with_education.py @@ -0,0 +1,32 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("people_with_education") + + # Read table from garden dataset. + tables = list(ds_garden) + + # + # Process data. + # + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=tables, check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/energy/2022-08-03/electricity_mix.py b/etl/steps/data/grapher/energy/2022-08-03/electricity_mix.py index b7466fd51ed..989d3d93cfe 100644 --- a/etl/steps/data/grapher/energy/2022-08-03/electricity_mix.py +++ b/etl/steps/data/grapher/energy/2022-08-03/electricity_mix.py @@ -1,5 +1,4 @@ -"""Grapher step for the Electricity Mix (BP & Ember, 2022) dataset. -""" +"""Grapher step for the Electricity Mix (BP & Ember, 2022) dataset.""" from owid import catalog diff --git a/etl/steps/data/grapher/energy/2022-12-12/electricity_mix.py b/etl/steps/data/grapher/energy/2022-12-12/electricity_mix.py index 2d0246d34ab..a7c55b3fec7 100644 --- a/etl/steps/data/grapher/energy/2022-12-12/electricity_mix.py +++ b/etl/steps/data/grapher/energy/2022-12-12/electricity_mix.py @@ -1,5 +1,4 @@ -"""Grapher step for the Electricity Mix (BP & Ember, 2022) dataset. -""" +"""Grapher step for the Electricity Mix (BP & Ember, 2022) dataset.""" from copy import deepcopy diff --git a/etl/steps/data/grapher/energy/2023-07-10/electricity_mix.py b/etl/steps/data/grapher/energy/2023-07-10/electricity_mix.py index 234900ffb45..8219e39c8e6 100644 --- a/etl/steps/data/grapher/energy/2023-07-10/electricity_mix.py +++ b/etl/steps/data/grapher/energy/2023-07-10/electricity_mix.py @@ -1,6 +1,4 @@ -"""Grapher step for the Electricity Mix (Energy Institute & Ember) dataset. - -""" +"""Grapher step for the Electricity Mix (Energy Institute & Ember) dataset.""" from owid.catalog import Dataset diff --git a/etl/steps/data/grapher/energy/2023-07-10/energy_mix.py b/etl/steps/data/grapher/energy/2023-07-10/energy_mix.py index 50efcb8c6ce..98c5629e8c4 100644 --- a/etl/steps/data/grapher/energy/2023-07-10/energy_mix.py +++ b/etl/steps/data/grapher/energy/2023-07-10/energy_mix.py @@ -1,5 +1,4 @@ -"""Grapher step for BP's energy mix dataset. -""" +"""Grapher step for BP's energy mix dataset.""" from owid.catalog import Dataset diff --git a/etl/steps/data/grapher/energy/2023-07-10/fossil_fuel_production.py b/etl/steps/data/grapher/energy/2023-07-10/fossil_fuel_production.py index 62cd3b209f9..3a46c0128ea 100644 --- a/etl/steps/data/grapher/energy/2023-07-10/fossil_fuel_production.py +++ b/etl/steps/data/grapher/energy/2023-07-10/fossil_fuel_production.py @@ -1,5 +1,4 @@ -"""Grapher step for the fossil fuel production dataset. -""" +"""Grapher step for the fossil fuel production dataset.""" from owid.catalog import Dataset diff --git a/etl/steps/data/grapher/energy/2023-07-10/fossil_fuel_reserves_production_ratio.py b/etl/steps/data/grapher/energy/2023-07-10/fossil_fuel_reserves_production_ratio.py index 066bd1a2e4b..c7c824244ff 100644 --- a/etl/steps/data/grapher/energy/2023-07-10/fossil_fuel_reserves_production_ratio.py +++ b/etl/steps/data/grapher/energy/2023-07-10/fossil_fuel_reserves_production_ratio.py @@ -1,6 +1,4 @@ -"""Grapher step for the fossil fuel reserves-to-production ratio dataset. - -""" +"""Grapher step for the fossil fuel reserves-to-production ratio dataset.""" from owid.catalog import Dataset diff --git a/etl/steps/data/grapher/energy/2023-07-10/global_primary_energy.py b/etl/steps/data/grapher/energy/2023-07-10/global_primary_energy.py index f5f10837270..65cf3e7ee03 100644 --- a/etl/steps/data/grapher/energy/2023-07-10/global_primary_energy.py +++ b/etl/steps/data/grapher/energy/2023-07-10/global_primary_energy.py @@ -1,5 +1,5 @@ -"""Grapher step for the global primary energy dataset. -""" +"""Grapher step for the global primary energy dataset.""" + from owid.catalog import Dataset from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/energy/2023-07-10/photovoltaic_cost_and_capacity.py b/etl/steps/data/grapher/energy/2023-07-10/photovoltaic_cost_and_capacity.py index 021c97cfe72..4bd2deee470 100644 --- a/etl/steps/data/grapher/energy/2023-07-10/photovoltaic_cost_and_capacity.py +++ b/etl/steps/data/grapher/energy/2023-07-10/photovoltaic_cost_and_capacity.py @@ -1,6 +1,4 @@ -"""Load garden dataset of photovoltaic cost and capacity and create a grapher dataset. - -""" +"""Load garden dataset of photovoltaic cost and capacity and create a grapher dataset.""" from owid.catalog import Dataset diff --git a/etl/steps/data/grapher/energy/2023-07-10/primary_energy_consumption.py b/etl/steps/data/grapher/energy/2023-07-10/primary_energy_consumption.py index 67ee91dc98d..f1016dcd078 100644 --- a/etl/steps/data/grapher/energy/2023-07-10/primary_energy_consumption.py +++ b/etl/steps/data/grapher/energy/2023-07-10/primary_energy_consumption.py @@ -1,5 +1,5 @@ -"""Grapher step for the primary energy consumption dataset. -""" +"""Grapher step for the primary energy consumption dataset.""" + from owid.catalog import Dataset from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/energy/2023-07-10/uk_historical_electricity.py b/etl/steps/data/grapher/energy/2023-07-10/uk_historical_electricity.py index 44e8121fa96..4d99bf2fa56 100644 --- a/etl/steps/data/grapher/energy/2023-07-10/uk_historical_electricity.py +++ b/etl/steps/data/grapher/energy/2023-07-10/uk_historical_electricity.py @@ -1,5 +1,5 @@ -"""Grapher step for the UK historical electricity dataset. -""" +"""Grapher step for the UK historical electricity dataset.""" + from owid.catalog import Dataset from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/energy/2023-12-12/electricity_mix.py b/etl/steps/data/grapher/energy/2023-12-12/electricity_mix.py index 29ff3a89233..b511938b549 100644 --- a/etl/steps/data/grapher/energy/2023-12-12/electricity_mix.py +++ b/etl/steps/data/grapher/energy/2023-12-12/electricity_mix.py @@ -1,7 +1,4 @@ -"""Grapher step for the Electricity Mix (Energy Institute & Ember) dataset. - -""" - +"""Grapher step for the Electricity Mix (Energy Institute & Ember) dataset.""" from etl.grapher_helpers import add_columns_for_multiindicator_chart from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/energy/2023-12-12/energy_mix.py b/etl/steps/data/grapher/energy/2023-12-12/energy_mix.py index 556686d7abe..4a4c5f56cc3 100644 --- a/etl/steps/data/grapher/energy/2023-12-12/energy_mix.py +++ b/etl/steps/data/grapher/energy/2023-12-12/energy_mix.py @@ -1,5 +1,5 @@ -"""Grapher step for BP's energy mix dataset. -""" +"""Grapher step for BP's energy mix dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/energy/2023-12-12/fossil_fuel_production.py b/etl/steps/data/grapher/energy/2023-12-12/fossil_fuel_production.py index 55ad9fc6420..fa3d18dc4e7 100644 --- a/etl/steps/data/grapher/energy/2023-12-12/fossil_fuel_production.py +++ b/etl/steps/data/grapher/energy/2023-12-12/fossil_fuel_production.py @@ -1,5 +1,5 @@ -"""Grapher step for the fossil fuel production dataset. -""" +"""Grapher step for the fossil fuel production dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/energy/2023-12-12/fossil_fuel_reserves_production_ratio.py b/etl/steps/data/grapher/energy/2023-12-12/fossil_fuel_reserves_production_ratio.py index 0277060bcbf..a67d125045f 100644 --- a/etl/steps/data/grapher/energy/2023-12-12/fossil_fuel_reserves_production_ratio.py +++ b/etl/steps/data/grapher/energy/2023-12-12/fossil_fuel_reserves_production_ratio.py @@ -1,6 +1,5 @@ -"""Grapher step for the fossil fuel reserves-to-production ratio dataset. +"""Grapher step for the fossil fuel reserves-to-production ratio dataset.""" -""" from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/energy/2023-12-12/global_primary_energy.py b/etl/steps/data/grapher/energy/2023-12-12/global_primary_energy.py index 4962fccada8..1fa296d643d 100644 --- a/etl/steps/data/grapher/energy/2023-12-12/global_primary_energy.py +++ b/etl/steps/data/grapher/energy/2023-12-12/global_primary_energy.py @@ -1,5 +1,5 @@ -"""Grapher step for the global primary energy dataset. -""" +"""Grapher step for the global primary energy dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/energy/2023-12-12/photovoltaic_cost_and_capacity.py b/etl/steps/data/grapher/energy/2023-12-12/photovoltaic_cost_and_capacity.py index cdf0553a357..adf46cacdc6 100644 --- a/etl/steps/data/grapher/energy/2023-12-12/photovoltaic_cost_and_capacity.py +++ b/etl/steps/data/grapher/energy/2023-12-12/photovoltaic_cost_and_capacity.py @@ -1,7 +1,4 @@ -"""Load garden dataset of photovoltaic cost and capacity and create a grapher dataset. - -""" - +"""Load garden dataset of photovoltaic cost and capacity and create a grapher dataset.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/energy/2023-12-12/primary_energy_consumption.py b/etl/steps/data/grapher/energy/2023-12-12/primary_energy_consumption.py index 906fc549bbe..740dd97e760 100644 --- a/etl/steps/data/grapher/energy/2023-12-12/primary_energy_consumption.py +++ b/etl/steps/data/grapher/energy/2023-12-12/primary_energy_consumption.py @@ -1,5 +1,5 @@ -"""Grapher step for the primary energy consumption dataset. -""" +"""Grapher step for the primary energy consumption dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/energy/2023-12-12/uk_historical_electricity.py b/etl/steps/data/grapher/energy/2023-12-12/uk_historical_electricity.py index e13f2e51f8d..32a2ea9bc88 100644 --- a/etl/steps/data/grapher/energy/2023-12-12/uk_historical_electricity.py +++ b/etl/steps/data/grapher/energy/2023-12-12/uk_historical_electricity.py @@ -1,5 +1,5 @@ -"""Grapher step for the UK historical electricity dataset. -""" +"""Grapher step for the UK historical electricity dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/energy/2024-05-08/electricity_mix.py b/etl/steps/data/grapher/energy/2024-05-08/electricity_mix.py index 29ff3a89233..b511938b549 100644 --- a/etl/steps/data/grapher/energy/2024-05-08/electricity_mix.py +++ b/etl/steps/data/grapher/energy/2024-05-08/electricity_mix.py @@ -1,7 +1,4 @@ -"""Grapher step for the Electricity Mix (Energy Institute & Ember) dataset. - -""" - +"""Grapher step for the Electricity Mix (Energy Institute & Ember) dataset.""" from etl.grapher_helpers import add_columns_for_multiindicator_chart from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/energy/2024-05-08/energy_mix.py b/etl/steps/data/grapher/energy/2024-05-08/energy_mix.py index 556686d7abe..4a4c5f56cc3 100644 --- a/etl/steps/data/grapher/energy/2024-05-08/energy_mix.py +++ b/etl/steps/data/grapher/energy/2024-05-08/energy_mix.py @@ -1,5 +1,5 @@ -"""Grapher step for BP's energy mix dataset. -""" +"""Grapher step for BP's energy mix dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/energy/2024-05-08/fossil_fuel_production.py b/etl/steps/data/grapher/energy/2024-05-08/fossil_fuel_production.py index 55ad9fc6420..fa3d18dc4e7 100644 --- a/etl/steps/data/grapher/energy/2024-05-08/fossil_fuel_production.py +++ b/etl/steps/data/grapher/energy/2024-05-08/fossil_fuel_production.py @@ -1,5 +1,5 @@ -"""Grapher step for the fossil fuel production dataset. -""" +"""Grapher step for the fossil fuel production dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/energy/2024-05-08/fossil_fuel_reserves_production_ratio.py b/etl/steps/data/grapher/energy/2024-05-08/fossil_fuel_reserves_production_ratio.py index 0277060bcbf..a67d125045f 100644 --- a/etl/steps/data/grapher/energy/2024-05-08/fossil_fuel_reserves_production_ratio.py +++ b/etl/steps/data/grapher/energy/2024-05-08/fossil_fuel_reserves_production_ratio.py @@ -1,6 +1,5 @@ -"""Grapher step for the fossil fuel reserves-to-production ratio dataset. +"""Grapher step for the fossil fuel reserves-to-production ratio dataset.""" -""" from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/energy/2024-05-08/global_primary_energy.py b/etl/steps/data/grapher/energy/2024-05-08/global_primary_energy.py index 4962fccada8..1fa296d643d 100644 --- a/etl/steps/data/grapher/energy/2024-05-08/global_primary_energy.py +++ b/etl/steps/data/grapher/energy/2024-05-08/global_primary_energy.py @@ -1,5 +1,5 @@ -"""Grapher step for the global primary energy dataset. -""" +"""Grapher step for the global primary energy dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/energy/2024-05-08/photovoltaic_cost_and_capacity.py b/etl/steps/data/grapher/energy/2024-05-08/photovoltaic_cost_and_capacity.py index cdf0553a357..adf46cacdc6 100644 --- a/etl/steps/data/grapher/energy/2024-05-08/photovoltaic_cost_and_capacity.py +++ b/etl/steps/data/grapher/energy/2024-05-08/photovoltaic_cost_and_capacity.py @@ -1,7 +1,4 @@ -"""Load garden dataset of photovoltaic cost and capacity and create a grapher dataset. - -""" - +"""Load garden dataset of photovoltaic cost and capacity and create a grapher dataset.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/energy/2024-05-08/primary_energy_consumption.py b/etl/steps/data/grapher/energy/2024-05-08/primary_energy_consumption.py index 7890ec21d9c..dc373d8dee8 100644 --- a/etl/steps/data/grapher/energy/2024-05-08/primary_energy_consumption.py +++ b/etl/steps/data/grapher/energy/2024-05-08/primary_energy_consumption.py @@ -1,5 +1,5 @@ -"""Grapher step for the primary energy consumption dataset. -""" +"""Grapher step for the primary energy consumption dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/energy/2024-05-08/uk_historical_electricity.py b/etl/steps/data/grapher/energy/2024-05-08/uk_historical_electricity.py index f6cf145c0c3..78bd90ae02f 100644 --- a/etl/steps/data/grapher/energy/2024-05-08/uk_historical_electricity.py +++ b/etl/steps/data/grapher/energy/2024-05-08/uk_historical_electricity.py @@ -1,5 +1,5 @@ -"""Grapher step for the UK historical electricity dataset. -""" +"""Grapher step for the UK historical electricity dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/energy/2024-06-20/electricity_mix.py b/etl/steps/data/grapher/energy/2024-06-20/electricity_mix.py index 29ff3a89233..b511938b549 100644 --- a/etl/steps/data/grapher/energy/2024-06-20/electricity_mix.py +++ b/etl/steps/data/grapher/energy/2024-06-20/electricity_mix.py @@ -1,7 +1,4 @@ -"""Grapher step for the Electricity Mix (Energy Institute & Ember) dataset. - -""" - +"""Grapher step for the Electricity Mix (Energy Institute & Ember) dataset.""" from etl.grapher_helpers import add_columns_for_multiindicator_chart from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/energy/2024-06-20/energy_mix.py b/etl/steps/data/grapher/energy/2024-06-20/energy_mix.py index 556686d7abe..4a4c5f56cc3 100644 --- a/etl/steps/data/grapher/energy/2024-06-20/energy_mix.py +++ b/etl/steps/data/grapher/energy/2024-06-20/energy_mix.py @@ -1,5 +1,5 @@ -"""Grapher step for BP's energy mix dataset. -""" +"""Grapher step for BP's energy mix dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/energy/2024-06-20/fossil_fuel_production.py b/etl/steps/data/grapher/energy/2024-06-20/fossil_fuel_production.py index 55ad9fc6420..fa3d18dc4e7 100644 --- a/etl/steps/data/grapher/energy/2024-06-20/fossil_fuel_production.py +++ b/etl/steps/data/grapher/energy/2024-06-20/fossil_fuel_production.py @@ -1,5 +1,5 @@ -"""Grapher step for the fossil fuel production dataset. -""" +"""Grapher step for the fossil fuel production dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/energy/2024-06-20/fossil_fuel_reserves_production_ratio.py b/etl/steps/data/grapher/energy/2024-06-20/fossil_fuel_reserves_production_ratio.py index 0277060bcbf..a67d125045f 100644 --- a/etl/steps/data/grapher/energy/2024-06-20/fossil_fuel_reserves_production_ratio.py +++ b/etl/steps/data/grapher/energy/2024-06-20/fossil_fuel_reserves_production_ratio.py @@ -1,6 +1,5 @@ -"""Grapher step for the fossil fuel reserves-to-production ratio dataset. +"""Grapher step for the fossil fuel reserves-to-production ratio dataset.""" -""" from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/energy/2024-06-20/global_primary_energy.py b/etl/steps/data/grapher/energy/2024-06-20/global_primary_energy.py index 4962fccada8..1fa296d643d 100644 --- a/etl/steps/data/grapher/energy/2024-06-20/global_primary_energy.py +++ b/etl/steps/data/grapher/energy/2024-06-20/global_primary_energy.py @@ -1,5 +1,5 @@ -"""Grapher step for the global primary energy dataset. -""" +"""Grapher step for the global primary energy dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/energy/2024-06-20/primary_energy_consumption.py b/etl/steps/data/grapher/energy/2024-06-20/primary_energy_consumption.py index 7890ec21d9c..dc373d8dee8 100644 --- a/etl/steps/data/grapher/energy/2024-06-20/primary_energy_consumption.py +++ b/etl/steps/data/grapher/energy/2024-06-20/primary_energy_consumption.py @@ -1,5 +1,5 @@ -"""Grapher step for the primary energy consumption dataset. -""" +"""Grapher step for the primary energy consumption dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/energy/2024-06-20/uk_historical_electricity.py b/etl/steps/data/grapher/energy/2024-06-20/uk_historical_electricity.py index e13f2e51f8d..32a2ea9bc88 100644 --- a/etl/steps/data/grapher/energy/2024-06-20/uk_historical_electricity.py +++ b/etl/steps/data/grapher/energy/2024-06-20/uk_historical_electricity.py @@ -1,5 +1,5 @@ -"""Grapher step for the UK historical electricity dataset. -""" +"""Grapher step for the UK historical electricity dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/energy/2024-10-30/photovoltaic_cost_and_capacity.py b/etl/steps/data/grapher/energy/2024-10-30/photovoltaic_cost_and_capacity.py index 15d3fb825d1..adf46cacdc6 100644 --- a/etl/steps/data/grapher/energy/2024-10-30/photovoltaic_cost_and_capacity.py +++ b/etl/steps/data/grapher/energy/2024-10-30/photovoltaic_cost_and_capacity.py @@ -1,6 +1,4 @@ -"""Load garden dataset of photovoltaic cost and capacity and create a grapher dataset. - -""" +"""Load garden dataset of photovoltaic cost and capacity and create a grapher dataset.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/energy/2024-11-01/photovoltaic_cost_and_capacity.py b/etl/steps/data/grapher/energy/2024-11-01/photovoltaic_cost_and_capacity.py index 15d3fb825d1..adf46cacdc6 100644 --- a/etl/steps/data/grapher/energy/2024-11-01/photovoltaic_cost_and_capacity.py +++ b/etl/steps/data/grapher/energy/2024-11-01/photovoltaic_cost_and_capacity.py @@ -1,6 +1,4 @@ -"""Load garden dataset of photovoltaic cost and capacity and create a grapher dataset. - -""" +"""Load garden dataset of photovoltaic cost and capacity and create a grapher dataset.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/energy/2024-11-15/photovoltaic_cost_and_capacity.py b/etl/steps/data/grapher/energy/2024-11-15/photovoltaic_cost_and_capacity.py index 15d3fb825d1..adf46cacdc6 100644 --- a/etl/steps/data/grapher/energy/2024-11-15/photovoltaic_cost_and_capacity.py +++ b/etl/steps/data/grapher/energy/2024-11-15/photovoltaic_cost_and_capacity.py @@ -1,6 +1,4 @@ -"""Load garden dataset of photovoltaic cost and capacity and create a grapher dataset. - -""" +"""Load garden dataset of photovoltaic cost and capacity and create a grapher dataset.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/energy/2024-11-20/energy_prices.py b/etl/steps/data/grapher/energy/2024-11-20/energy_prices.py new file mode 100644 index 00000000000..ed9db95f813 --- /dev/null +++ b/etl/steps/data/grapher/energy/2024-11-20/energy_prices.py @@ -0,0 +1,22 @@ +"""Load garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # Load tables from garden dataset. + ds_garden = paths.load_dataset("energy_prices") + tb_annual = ds_garden.read("energy_prices_annual", reset_index=False) + tb_monthly = ds_garden.read("energy_prices_monthly", reset_index=False) + + # Create a new grapher dataset. + dataset = create_dataset( + dest_dir=dest_dir, + tables=[tb_annual, tb_monthly], + check_variables_metadata=True, + default_metadata=ds_garden.metadata, + ) + dataset.save() diff --git a/etl/steps/data/grapher/faostat/2022-05-17/shared.py b/etl/steps/data/grapher/faostat/2022-05-17/shared.py index aa3c05c4bab..07aa8b95d4c 100644 --- a/etl/steps/data/grapher/faostat/2022-05-17/shared.py +++ b/etl/steps/data/grapher/faostat/2022-05-17/shared.py @@ -1,6 +1,4 @@ -"""Common grapher step for all FAOSTAT domains. - -""" +"""Common grapher step for all FAOSTAT domains.""" from pathlib import Path diff --git a/etl/steps/data/grapher/faostat/2023-02-22/shared.py b/etl/steps/data/grapher/faostat/2023-02-22/shared.py index f0245ecb84e..3338e2f46db 100644 --- a/etl/steps/data/grapher/faostat/2023-02-22/shared.py +++ b/etl/steps/data/grapher/faostat/2023-02-22/shared.py @@ -1,6 +1,5 @@ -"""Common grapher step for all FAOSTAT domains. +"""Common grapher step for all FAOSTAT domains.""" -""" from pathlib import Path from owid.catalog import Dataset diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_cahd.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_cahd.py index eb6916c2187..446350a168f 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_cahd.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_cahd.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_cahd dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_ef.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_ef.py index 304863c1b67..079b62e82f1 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_ef.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_ef.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_ef dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_ei.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_ei.py index d5ca840e309..5a26417991b 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_ei.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_ei.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_ei dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_ek.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_ek.py index 1438bef60af..cd9a22cc4b9 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_ek.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_ek.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_ek dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_el.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_el.py index dad80439d5f..a250fe72e4b 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_el.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_el.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_el dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_emn.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_emn.py index a4ec711b24f..4889d38b3ce 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_emn.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_emn.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_emn dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_ep.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_ep.py index 425db60fd08..a390ea767e6 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_ep.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_ep.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_ep dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_esb.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_esb.py index 9443efd4c2f..95386e71283 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_esb.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_esb.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_esb dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_fa.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_fa.py index 68c503b33fb..cc0a0a828c7 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_fa.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_fa.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_fa dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_fbsc.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_fbsc.py index a96693ea59e..96497370c92 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_fbsc.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_fbsc.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_fbsc dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_fo.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_fo.py index 52d47d9693b..a06f7821a6c 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_fo.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_fo.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_fo dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_fs.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_fs.py index 9ac98d46d8b..0e8413b983f 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_fs.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_fs.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_fs dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_gn.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_gn.py index dd97adb515e..b00af192b6c 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_gn.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_gn.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_gn dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_ic.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_ic.py index 3bb8b297f9b..f13c6ae77ed 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_ic.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_ic.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_ic dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_lc.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_lc.py index 9e55fe697eb..6a4f626f51b 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_lc.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_lc.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_lc dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_qcl.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_qcl.py index 17ea29863b0..c73003a91cc 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_qcl.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_qcl.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_qcl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_qi.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_qi.py index ec1e351be6d..8d1e7a631c3 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_qi.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_qi.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_qi dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_qv.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_qv.py index a8ad501a473..29bd80d2929 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_qv.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_qv.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_qv dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_rfb.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_rfb.py index 9203ba8a494..5e1bc67635e 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_rfb.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_rfb.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_rfb dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_rfn.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_rfn.py index 006af8bb6ce..28d1d9360e9 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_rfn.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_rfn.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_rfn dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_rl.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_rl.py index 95550785095..a2ea1ded775 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_rl.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_rl.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_rl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_rp.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_rp.py index b552b8f0035..70112163cfc 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_rp.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_rp.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_rp dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_rt.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_rt.py index 709d69ac2d5..1d4bf568fcb 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_rt.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_rt.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_rt dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_scl.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_scl.py index 95725e189c9..4d421590057 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_scl.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_scl.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_scl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_sdgb.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_sdgb.py index 7e0187e34ba..bf4d2252f63 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_sdgb.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_sdgb.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_sdgb dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_tcl.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_tcl.py index 0babbf0b2e3..a055ad45bb1 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_tcl.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_tcl.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_tcl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_ti.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_ti.py index 94634f7e505..7c33b6cf5e8 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_ti.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_ti.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_ti dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/faostat_wcad.py b/etl/steps/data/grapher/faostat/2023-06-12/faostat_wcad.py index 39ee4dcf7f0..21334f37bea 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/faostat_wcad.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/faostat_wcad.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_wcad dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2023-06-12/shared.py b/etl/steps/data/grapher/faostat/2023-06-12/shared.py index f0245ecb84e..3338e2f46db 100644 --- a/etl/steps/data/grapher/faostat/2023-06-12/shared.py +++ b/etl/steps/data/grapher/faostat/2023-06-12/shared.py @@ -1,6 +1,5 @@ -"""Common grapher step for all FAOSTAT domains. +"""Common grapher step for all FAOSTAT domains.""" -""" from pathlib import Path from owid.catalog import Dataset diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_cahd.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_cahd.py index eb6916c2187..446350a168f 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_cahd.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_cahd.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_cahd dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_ei.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_ei.py index d5ca840e309..5a26417991b 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_ei.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_ei.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_ei dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_ek.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_ek.py index 1438bef60af..cd9a22cc4b9 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_ek.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_ek.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_ek dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_emn.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_emn.py index a4ec711b24f..4889d38b3ce 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_emn.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_emn.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_emn dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_esb.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_esb.py index 9443efd4c2f..95386e71283 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_esb.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_esb.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_esb dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_fa.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_fa.py index 68c503b33fb..cc0a0a828c7 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_fa.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_fa.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_fa dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_fbsc.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_fbsc.py index a96693ea59e..96497370c92 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_fbsc.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_fbsc.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_fbsc dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_fo.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_fo.py index 52d47d9693b..a06f7821a6c 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_fo.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_fo.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_fo dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_fs.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_fs.py index 9ac98d46d8b..0e8413b983f 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_fs.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_fs.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_fs dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_ic.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_ic.py index 3bb8b297f9b..f13c6ae77ed 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_ic.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_ic.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_ic dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_lc.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_lc.py index 9e55fe697eb..6a4f626f51b 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_lc.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_lc.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_lc dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_qcl.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_qcl.py index 17ea29863b0..c73003a91cc 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_qcl.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_qcl.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_qcl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_qi.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_qi.py index ec1e351be6d..8d1e7a631c3 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_qi.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_qi.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_qi dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_qv.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_qv.py index a8ad501a473..29bd80d2929 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_qv.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_qv.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_qv dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_rfb.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_rfb.py index 9203ba8a494..5e1bc67635e 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_rfb.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_rfb.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_rfb dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_rfn.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_rfn.py index 006af8bb6ce..28d1d9360e9 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_rfn.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_rfn.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_rfn dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_rl.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_rl.py index 95550785095..a2ea1ded775 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_rl.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_rl.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_rl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_rp.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_rp.py index b552b8f0035..70112163cfc 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_rp.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_rp.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_rp dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_rt.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_rt.py index 709d69ac2d5..1d4bf568fcb 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_rt.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_rt.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_rt dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_scl.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_scl.py index 95725e189c9..4d421590057 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_scl.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_scl.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_scl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_sdgb.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_sdgb.py index 7e0187e34ba..bf4d2252f63 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_sdgb.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_sdgb.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_sdgb dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_tcl.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_tcl.py index 0babbf0b2e3..a055ad45bb1 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_tcl.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_tcl.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_tcl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/faostat_ti.py b/etl/steps/data/grapher/faostat/2024-03-14/faostat_ti.py index 94634f7e505..7c33b6cf5e8 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/faostat_ti.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/faostat_ti.py @@ -1,2 +1,3 @@ """FAOSTAT grapher step for faostat_ti dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/grapher/faostat/2024-03-14/shared.py b/etl/steps/data/grapher/faostat/2024-03-14/shared.py index bf7d941e000..60dc7a0156d 100644 --- a/etl/steps/data/grapher/faostat/2024-03-14/shared.py +++ b/etl/steps/data/grapher/faostat/2024-03-14/shared.py @@ -1,6 +1,5 @@ -"""Common grapher step for all FAOSTAT domains. +"""Common grapher step for all FAOSTAT domains.""" -""" from pathlib import Path from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/fasttrack/2022-11-01/lighting_efficiency_uk.meta.yml b/etl/steps/data/grapher/fasttrack/2022-11-01/lighting_efficiency_uk.meta.yml index af141edea71..eff3194f072 100644 --- a/etl/steps/data/grapher/fasttrack/2022-11-01/lighting_efficiency_uk.meta.yml +++ b/etl/steps/data/grapher/fasttrack/2022-11-01/lighting_efficiency_uk.meta.yml @@ -1,40 +1,18 @@ dataset: - namespace: fasttrack - version: '2022-11-01' - short_name: lighting_efficiency_uk title: Lighting effiency and shares in the UK description: '' - sources: - - name: 'Fouquet & Pearson (2006). Seven centuries of energy services: The price - and use of light in the United Kingdom (1300-2000).' - published_by: 'Fouquet, R., & Pearson, P. J. (2006). Seven centuries of energy - services: The price and use of light in the United Kingdom (1300-2000). The - energy journal, 27(1).' - url: https://www.jstor.org/stable/23296980 + licenses: + - {} tables: lighting_efficiency_uk: variables: share_of_lighting_uk: title: share_of_lighting_uk - short_unit: '%' unit: '%' + short_unit: '%' description: The share of lighting in the UK that was provided by each source. - sources: - - name: 'Fouquet & Pearson (2006). Seven centuries of energy services: The - price and use of light in the United Kingdom (1300-2000).' - published_by: 'Fouquet, R., & Pearson, P. J. (2006). Seven centuries of - energy services: The price and use of light in the United Kingdom (1300-2000). - The energy journal, 27(1).' - url: https://www.jstor.org/stable/23296980 efficiency_lighting_uk: title: efficiency_lighting_uk unit: lumen-hours per kWh - description: The efficiency of lighting measures the output of light per unit - of energy. It's measured in lumen-hours per kilowatt-hour (kWh). - sources: - - name: 'Fouquet & Pearson (2006). Seven centuries of energy services: The - price and use of light in the United Kingdom (1300-2000).' - published_by: 'Fouquet, R., & Pearson, P. J. (2006). Seven centuries of - energy services: The price and use of light in the United Kingdom (1300-2000). - The energy journal, 27(1).' - url: https://www.jstor.org/stable/23296980 + description: |- + The efficiency of lighting measures the output of light per unit of energy. It's measured in lumen-hours per kilowatt-hour (kWh). diff --git a/etl/steps/data/grapher/fasttrack/2022-11-01/lighting_efficiency_uk.py b/etl/steps/data/grapher/fasttrack/2022-11-01/lighting_efficiency_uk.py index db8277a1af8..019afbe6fc1 100644 --- a/etl/steps/data/grapher/fasttrack/2022-11-01/lighting_efficiency_uk.py +++ b/etl/steps/data/grapher/fasttrack/2022-11-01/lighting_efficiency_uk.py @@ -1,21 +1,40 @@ import pandas as pd -from owid import catalog -from etl.helpers import PathFinder +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot -N = PathFinder(__file__) +paths = PathFinder(__file__) def run(dest_dir: str) -> None: # load snapshot - data = pd.read_csv(Snapshot("fasttrack/2022-11-01/lighting_efficiency_uk.csv").path) + snap = Snapshot("fasttrack/2022-11-01/lighting_efficiency_uk.csv") - # create empty dataframe and table - ds = catalog.Dataset.create_empty(dest_dir) - tb = catalog.Table(data, short_name=N.short_name) + # load data + tb = snap.read_csv() + + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) # add table, update metadata from *.meta.yml and save - ds.add(tb) - ds.update_metadata(N.metadata_path) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/2023-03-27/global_warming_contributions.meta.yml b/etl/steps/data/grapher/fasttrack/2023-03-27/global_warming_contributions.meta.yml new file mode 100644 index 00000000000..d937637adbd --- /dev/null +++ b/etl/steps/data/grapher/fasttrack/2023-03-27/global_warming_contributions.meta.yml @@ -0,0 +1,255 @@ +dataset: + title: Global warming contributions + description: |- + Jones et al. (2023) quantify national and regional contributions to the increase of global mean surface temperature over the last few centuries. As they detail: the "dataset describing the global warming response to national emissions CO2, CH4 and N2O from fossil and land use sources during 1851-2021. + + National CO2 emissions data are collated from the Global Carbon Project (Andrew and Peters, 2022; Friedlingstein et al., 2022). + + National CH4 and N2O emissions data are collated from PRIMAP-hist (HISTTP) (Gütschow et al., 2022). + + We construct a time series of cumulative CO2-equivalent emissions for each country, gas, and emissions source (fossil or land use). Emissions of CH4 and N2O emissions are related to cumulative CO2-equivalent emissions using the Global Warming Potential (GWP*) approach, with best-estimates of the coefficients taken from the IPCC AR6 (Forster et al., 2021). + + Warming in response to cumulative CO2-equivalent emissions is estimated using the transient climate response to cumulative carbon emissions (TCRE) approach, with best-estimate value of TCRE taken from the IPCC AR6 (Forster et al., 2021, Canadell et al., 2021). 'Warming' is specifically the change in global mean surface temperature (GMST)." + licenses: + - {} +tables: + global_warming_contributions: + variables: + annual_fossil_co2: + title: Annual fossil CO2 emissions + unit: tonnes + short_unit: t + annual_land_co2: + title: Annual CO2 emissions from agriculture and land use + unit: tonnes + short_unit: t + annual_co2: + title: Annual CO2 emissions + unit: tonnes + short_unit: t + annual_fossil_ch4: + title: annual_fossil_ch4 + unit: tonnes + short_unit: t + annual_land_ch4: + title: annual_land_ch4 + unit: tonnes + short_unit: t + annual_ch4: + title: annual_ch4 + unit: tonnes + short_unit: t + annual_fossil_n2o: + title: annual_fossil_n2o + unit: tonnes + short_unit: t + annual_land_n2o: + title: annual_land_n2o + unit: tonnes + short_unit: t + annual_n2o: + title: annual_n2o + unit: tonnes + short_unit: t + annual_fossil_ch4_co2eq: + title: Annual methane emissions from fossil fuels and industry + unit: tonnes + short_unit: t + description: |- + Methane emissions are calculated by Our World in Data based on emissions data from Jones et al. (2023) and IPCC conversion factors. Jones et al. (2023) give methane emissions in standard metric tonnes per year. We have converted these emissions to carbon-dioxide equivalents over a 100-year timescale using a conversion factor of 29.8 for fossil sources, and 27.2 for agricultural and land use sources (as per the IPCC AR6 report). + annual_land_ch4_co2eq: + title: Annual methane emissions from agriculture and land use + unit: tonnes + short_unit: t + description: |- + Methane emissions are calculated by Our World in Data based on emissions data from Jones et al. (2023) and IPCC conversion factors. Jones et al. (2023) give methane emissions in standard metric tonnes per year. We have converted these emissions to carbon-dioxide equivalents over a 100-year timescale using a conversion factor of 29.8 for fossil sources, and 27.2 for agricultural and land use sources (as per the IPCC AR6 report). + annual_ch4_co2eq: + title: Annual methane emissions + unit: tonnes + short_unit: t + description: |- + Methane emissions are calculated by Our World in Data based on emissions data from Jones et al. (2023) and IPCC conversion factors. Jones et al. (2023) give methane emissions in standard metric tonnes per year. We have converted these emissions to carbon-dioxide equivalents over a 100-year timescale using a conversion factor of 29.8 for fossil sources, and 27.2 for agricultural and land use sources (as per the IPCC AR6 report). + annual_fossil_n2o_co2eq: + title: Annual nitrous oxide emissions from fossil fuels and industry + unit: tonnes + short_unit: t + description: |- + Nitrous oxide emissions are calculated by Our World in Data based on emissions data from Jones et al. (2023) and IPCC AR6 conversion factors. Jones et al. (2023) give nitrous oxide emissions in standard metric tonnes per year. We have converted these emissions to carbon-dioxide equivalents over a 100-year timescale using a conversion factor of 273 (as per the IPCC AR6 report). + annual_land_n2o_co2eq: + title: Annual nitrous oxide emissions from agriculture and land use + unit: tonnes + short_unit: t + description: |- + Nitrous oxide emissions are calculated by Our World in Data based on emissions data from Jones et al. (2023) and IPCC AR6 conversion factors. Jones et al. (2023) give nitrous oxide emissions in standard metric tonnes per year. We have converted these emissions to carbon-dioxide equivalents over a 100-year timescale using a conversion factor of 273 (as per the IPCC AR6 report). + annual_n2o_co2eq: + title: Annual nitrous oxide emissions + unit: tonnes + short_unit: t + description: |- + Nitrous oxide emissions are calculated by Our World in Data based on emissions data from Jones et al. (2023) and IPCC AR6 conversion factors. Jones et al. (2023) give nitrous oxide emissions in standard metric tonnes per year. We have converted these emissions to carbon-dioxide equivalents over a 100-year timescale using a conversion factor of 273 (as per the IPCC AR6 report). + annual_ghg_co2eq: + title: Annual greenhouse gas emissions + unit: tonnes + short_unit: t + description: |- + Greenhouse gas emissions are calculated by Our World in Data based on emissions data from Jones et al. (2023) and IPCC AR6 conversion factors. Jones et al. (2023) give methane and nitrous oxide emissions in standard metric tonnes per year. We have converted these emissions to carbon-dioxide equivalents over a 100-year timescale using a conversion factor of 273 for nitrous oxide, 29.8 for methane from fossil sources, and 27.2 for methane from agricultural and land use sources (as per the IPCC AR6 report). + annual_fossil_co2eq: + title: Annual greenhouse gas emissions from fossil fuels and industry + unit: tonnes + short_unit: t + description: |- + Greenhouse gas emissions are calculated by Our World in Data based on emissions data from Jones et al. (2023) and IPCC AR6 conversion factors. Jones et al. (2023) give methane and nitrous oxide emissions in standard metric tonnes per year. We have converted these emissions to carbon-dioxide equivalents over a 100-year timescale using a conversion factor of 273 for nitrous oxide, 29.8 for methane from fossil sources, and 27.2 for methane from agricultural and land use sources (as per the IPCC AR6 report). + annual_land_co2eq: + title: Annual greenhouse gas emissions from agriculture and land use + unit: tonnes + short_unit: t + description: |- + Greenhouse gas emissions are calculated by Our World in Data based on emissions data from Jones et al. (2023) and IPCC AR6 conversion factors. Jones et al. (2023) give methane and nitrous oxide emissions in standard metric tonnes per year. We have converted these emissions to carbon-dioxide equivalents over a 100-year timescale using a conversion factor of 273 for nitrous oxide, 29.8 for methane from fossil sources, and 27.2 for methane from agricultural and land use sources (as per the IPCC AR6 report). + share_global_ch4: + title: Share of global methane emissions + unit: '%' + short_unit: '%' + description: |- + Methane emissions are calculated by Our World in Data based on emissions data from Jones et al. (2023) and IPCC conversion factors. Jones et al. (2023) give methane emissions in standard metric tonnes per year. We have converted these emissions to carbon-dioxide equivalents over a 100-year timescale using a conversion factor of 29.8 for fossil sources, and 27.2 for agricultural and land use sources (as per the IPCC AR6 report). + share_global_n2o: + title: Share of global nitrous oxide emissions + unit: '%' + short_unit: '%' + description: |- + Nitrous oxide emissions are calculated by Our World in Data based on emissions data from Jones et al. (2023) and IPCC AR6 conversion factors. Jones et al. (2023) give nitrous oxide emissions in standard metric tonnes per year. We have converted these emissions to carbon-dioxide equivalents over a 100-year timescale using a conversion factor of 273 (as per the IPCC AR6 report). + share_global_ghg: + title: Share of global greenhouse gas emissions + unit: '%' + short_unit: '%' + description: |- + Greenhouse gas emissions are calculated by Our World in Data based on emissions data from Jones et al. (2023) and IPCC AR6 conversion factors. Jones et al. (2023) give methane and nitrous oxide emissions in standard metric tonnes per year. We have converted these emissions to carbon-dioxide equivalents over a 100-year timescale using a conversion factor of 273 for nitrous oxide, 29.8 for methane from fossil sources, and 27.2 for methane from agricultural and land use sources (as per the IPCC AR6 report). + cumulative_fossil_co2: + title: cumulative_fossil_co2 + unit: tonnes + short_unit: t + cumulative_land_co2: + title: cumulative_land_co2 + unit: tonnes + short_unit: t + cumulative_co2: + title: cumulative_co2 + unit: tonnes + short_unit: t + cumulative_fossil_ch4: + title: cumulative_fossil_ch4 + unit: tonnes + short_unit: t + cumulative_land_ch4: + title: cumulative_land_ch4 + unit: tonnes + short_unit: t + cumulative_ch4: + title: cumulative_ch4 + unit: tonnes + short_unit: t + cumulative_fossil_n2o: + title: cumulative_fossil_n2o + unit: tonnes + short_unit: t + cumulative_land_n2o: + title: cumulative_land_n2o + unit: tonnes + short_unit: t + cumulative_n2o: + title: cumulative_n2o + unit: tonnes + short_unit: t + cumulative_fossil_ghg: + title: cumulative_fossil_ghg + unit: tonnes + short_unit: t + cumulative_land_ghg: + title: cumulative_land_ghg + unit: tonnes + short_unit: t + cumulative_ghg: + title: Cumulative greenhouse gas emissions + unit: tonnes + short_unit: t + description: |- + Greenhouse gas emissions are calculated by Our World in Data based on emissions data from Jones et al. (2023) and IPCC AR6 conversion factors. Jones et al. (2023) give methane and nitrous oxide emissions in standard metric tonnes per year. We have converted these emissions to carbon-dioxide equivalents over a 100-year timescale using a conversion factor of 273 for nitrous oxide, 29.8 for methane from fossil sources, and 27.2 for methane from agricultural and land use sources (as per the IPCC AR6 report). + temp_fossil_co2: + title: temp_fossil_co2 + unit: °C + short_unit: °C + temp_land_co2: + title: temp_land_co2 + unit: °C + short_unit: °C + temp_co2: + title: Change in global mean surface temperature from CO2 emissions + unit: °C + short_unit: °C + description: |- + This measures each country's contribution to global mean surface temperature (GMST) rise from its cumulative emissions of carbon dioxide. The warming effects of each gas are calculated based on cumulative CO2-equivalent emissions using the Global Warming Potential (GWP*) approach. + temp_fossil_ch4: + title: temp_fossil_ch4 + unit: °C + short_unit: °C + temp_land_ch4: + title: temp_land_ch4 + unit: °C + short_unit: °C + temp_ch4: + title: Change in global mean surface temperature from methane emissions + unit: °C + short_unit: °C + description: |- + This measures each country's contribution to global mean surface temperature (GMST) rise from its cumulative emissions of methane. The warming effects of each gas are calculated based on cumulative CO2-equivalent emissions using the Global Warming Potential (GWP*) approach. + temp_fossil_n2o: + title: temp_fossil_n2o + unit: °C + short_unit: °C + temp_land_n2o: + title: temp_land_n2o + unit: °C + short_unit: °C + temp_n2o: + title: Change in global mean surface temperature from nitrous oxide emissions + unit: °C + short_unit: °C + description: |- + This measures each country's contribution to global mean surface temperature (GMST) rise from its cumulative nitrous oxide emissions. The warming effects of each gas are calculated based on cumulative CO2-equivalent emissions using the Global Warming Potential (GWP*) approach. + temp_fossil_ghg: + title: temp_fossil_ghg + unit: °C + short_unit: °C + temp_land_ghg: + title: temp_land_ghg + unit: °C + short_unit: °C + temp_ghg: + title: Change in global mean surface temperature from greenhouse gas emissions + unit: °C + short_unit: °C + description: |- + This measures each country's contribution to global mean surface temperature (GMST) rise from its cumulative emissions of carbon dioxide, methane and nitrous oxide. The warming effects of each gas are calculated based on cumulative CO2-equivalent emissions using the Global Warming Potential (GWP*) approach. + share_global_warming: + title: Share of contribution to global warming + unit: '%' + short_unit: '%' + description: |- + This measures each country's contribution to global mean surface temperature (GMST) rise from its cumulative emissions of carbon dioxide, methane and nitrous oxide. The warming effects of each gas are calculated based on cumulative CO2-equivalent emissions using the Global Warming Potential (GWP*) approach. + annual_ch4_per_capita: + title: Methane emissions per person + unit: tonnes + short_unit: t + description: |- + Methane emissions per person are calculated by Our World in Data based on emissions data from Jones et al. (2023) and population data from HYDE and the UN World Population Prospects. Jones et al. (2023) give methane emissions in standard metric tonnes per year. We have converted these emissions to carbon-dioxide equivalents over a 100-year timescale using a conversion factor of 29.8 for fossil sources, and 27.2 for agricultural and land use sources (as per the IPCC AR6 report). + annual_n2o_per_capita: + title: Nitrous oxide emissions per person + unit: tonnes + short_unit: t + description: |- + Nitrous oxide emissions per person are calculated by Our World in Data based on emissions data from Jones et al. (2023) and population data from HYDE and the UN World Population Prospects. Jones et al. (2023) give nitrous oxide emissions in standard metric tonnes per year. We have converted these emissions to carbon-dioxide equivalents over a 100-year timescale using a conversion factor of 273 (as per the IPCC AR6 report). + annual_ghg_co2eq_per_capita: + title: Greenhouse gas emissions per person + unit: tonnes + short_unit: t + description: |- + Greenhouse gas emissions per person are calculated by Our World in Data based on emissions data from Jones et al. (2023) and population data from HYDE and the UN World Population Prospects. Jones et al. (2023) give methane and nitrous oxide emissions in standard metric tonnes per year. We have converted these emissions to carbon-dioxide equivalents over a 100-year timescale using a conversion factor of 273 for nitrous oxide, 29.8 for methane from fossil sources, and 27.2 for methane from agricultural and land use sources (as per the IPCC AR6 report). diff --git a/etl/steps/data/grapher/fasttrack/2023-03-27/global_warming_contributions.py b/etl/steps/data/grapher/fasttrack/2023-03-27/global_warming_contributions.py new file mode 100644 index 00000000000..cc079edc61d --- /dev/null +++ b/etl/steps/data/grapher/fasttrack/2023-03-27/global_warming_contributions.py @@ -0,0 +1,40 @@ +import pandas as pd + +from etl.helpers import PathFinder, create_dataset, get_metadata_path +from etl.snapshot import Snapshot + +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # load snapshot + snap = Snapshot("fasttrack/2023-03-27/global_warming_contributions.csv") + + # load data + tb = snap.read_csv() + + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) + + # add table, update metadata from *.meta.yml and save + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/2023-04-30/paratz.meta.yml b/etl/steps/data/grapher/fasttrack/2023-04-30/paratz.meta.yml index fa6dbc58bf9..1f55ca2acad 100644 --- a/etl/steps/data/grapher/fasttrack/2023-04-30/paratz.meta.yml +++ b/etl/steps/data/grapher/fasttrack/2023-04-30/paratz.meta.yml @@ -1,21 +1,16 @@ dataset: - sources: - - name: Paratz et al., (2023) - published_by: Heart Rhythm Journal - publication_year: 2023 - date_accessed: 2023-07-13 - url: https://www.heartrhythmjournal.com/article/S1547-5271(23)00027-9/fulltext + title: A systematic review of global autopsy rates in all-cause mortality and young sudden death, Paratz et al (2023) + description: |- + The data for this indicator is taken from: Paratz ED, Rowe SJ, Stub D, Pflaumer A, La Gerche A. A systematic review of global autopsy rates in all-cause mortality and young sudden death. Heart Rhythm. 2023 Apr;20(4):607-613. doi: 10.1016/j.hrthm.2023.01.008. + + The data is collated from a number of published papers and databases. The year shown reflects the date given in the database or the year of the publication. For Spain and Australia the data is only representative of a region of each country, Catalonia and Victoria, respectively. + licenses: + - {} tables: paratz: variables: autopsy_rate: title: Autopsy rate - short_unit: '%' unit: '%' + short_unit: '%' description: Autopsy rates reported in all-cause death. - sources: - - name: Paratz et al., (2023) - published_by: Heart Rhythm Journal - publication_year: 2023 - date_accessed: 2023-07-13 - url: https://www.heartrhythmjournal.com/article/S1547-5271(23)00027-9/fulltext diff --git a/etl/steps/data/grapher/fasttrack/2023-04-30/paratz.py b/etl/steps/data/grapher/fasttrack/2023-04-30/paratz.py index 349d0e134fd..114719618eb 100644 --- a/etl/steps/data/grapher/fasttrack/2023-04-30/paratz.py +++ b/etl/steps/data/grapher/fasttrack/2023-04-30/paratz.py @@ -1,10 +1,9 @@ import pandas as pd -from owid import catalog -from etl.helpers import PathFinder, create_dataset +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot -P = PathFinder(__file__) +paths = PathFinder(__file__) def run(dest_dir: str) -> None: @@ -12,11 +11,30 @@ def run(dest_dir: str) -> None: snap = Snapshot("fasttrack/2023-04-30/paratz.csv") # load data - data = pd.read_csv(snap.path) + tb = snap.read_csv() - # create empty dataframe and table - tb = catalog.Table(data, short_name=P.short_name) + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) # add table, update metadata from *.meta.yml and save ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/2023-05-03/apms_2014.meta.yml b/etl/steps/data/grapher/fasttrack/2023-05-03/apms_2014.meta.yml index b4c0e24ead3..21de647880e 100644 --- a/etl/steps/data/grapher/fasttrack/2023-05-03/apms_2014.meta.yml +++ b/etl/steps/data/grapher/fasttrack/2023-05-03/apms_2014.meta.yml @@ -1,24 +1,10 @@ dataset: - namespace: fasttrack - version: '2023-05-03' - short_name: apms_2014 title: Current depression in England by age and gender (APMS, 2014) - description: This is a dataset of the prevalence of current depression in the general - population in England, living in private households. Households were sampled randomly - and individuals were interviewed using the revised Clinical Interview Schedule - (CIS-R), which is a diagnostic structured interview format to determine whether - people had common mental disorders in the past week. In this dataset, presence - of a current episode of major depression was determined. - sources: - - name: Adult Psychiatric Morbidity Survey 2014, England (2016) - published_by: '"McManus S, Bebbington P, Jenkins R, Brugha T. (eds.) (2016) Mental - health and wellbeing in England: Adult Psychiatric Morbidity Survey 2014. Leeds: - NHS Digital"' - description: Surveys of individuals in randomly-selected private households - in England - publication_year: 2016 - date_accessed: 2022-12-01 - url: https://www.gov.uk/government/statistics/adult-psychiatric-morbidity-survey-mental-health-and-wellbeing-england-2014 + description: |- + This is a dataset of the prevalence of current depression in the general population in England, living in private households. Households were sampled randomly and individuals were interviewed using the revised Clinical Interview Schedule (CIS-R), which is a diagnostic structured interview format to determine whether people had common mental disorders in the past week. In this dataset, presence of a current episode of major depression was determined. + licenses: + - name: Open Government Licence v3.0 + url: https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/ tables: apms_2014: variables: diff --git a/etl/steps/data/grapher/fasttrack/2023-05-03/apms_2014.py b/etl/steps/data/grapher/fasttrack/2023-05-03/apms_2014.py index 07dbcf3e545..a1bcca99f8d 100644 --- a/etl/steps/data/grapher/fasttrack/2023-05-03/apms_2014.py +++ b/etl/steps/data/grapher/fasttrack/2023-05-03/apms_2014.py @@ -1,19 +1,40 @@ import pandas as pd -from owid import catalog -from etl.helpers import PathFinder, create_dataset +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot -P = PathFinder(__file__) +paths = PathFinder(__file__) def run(dest_dir: str) -> None: # load snapshot - data = pd.read_csv(Snapshot("fasttrack/2023-05-03/apms_2014.csv").path) + snap = Snapshot("fasttrack/2023-05-03/apms_2014.csv") - # create empty dataframe and table - tb = catalog.Table(data, short_name=P.short_name) + # load data + tb = snap.read_csv() + + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) # add table, update metadata from *.meta.yml and save - ds = create_dataset(dest_dir, tables=[tb]) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/2023-05-31/cholera.meta.yml b/etl/steps/data/grapher/fasttrack/2023-05-31/cholera.meta.yml index 7b2b37923d5..0be2698a83e 100644 --- a/etl/steps/data/grapher/fasttrack/2023-05-31/cholera.meta.yml +++ b/etl/steps/data/grapher/fasttrack/2023-05-31/cholera.meta.yml @@ -1,7 +1,4 @@ dataset: - namespace: fasttrack - version: '2023-05-31' - short_name: cholera title: Cholera reported cases, deaths and case fatality rate (WHO, 2023) description: |- The data is created by combining multiple WHO Weekly Epidemiological Reports for cholera reported cases, deaths and case fatality rate. @@ -17,37 +14,22 @@ dataset: 2020: https://web.archive.org/web/20230326231135/http://apps.who.int/iris/bitstream/handle/10665/345271/WER9637-445-454-eng-fre.pdf?sequence=1&isAllowed=y 2021: https://web.archive.org/web/20230526223955/https://apps.who.int/iris/bitstream/handle/10665/362858/WER9737-453-464-eng-fre.pdf?sequence=1&isAllowed=y - sources: - - name: World Health Organization (2023) - published_by: World Health Organization - date_accessed: 2023-05-31 + licenses: + - {} tables: cholera: variables: cholera_reported_cases: title: Cholera reported cases unit: reported cases - description: Confirmed cholera cases, including those confirmed clinically, - epidemiologically, or by laboratory investigation. - sources: - - name: World Health Organization (2023) - published_by: World Health Organization - date_accessed: 2023-05-31 + description: Confirmed cholera cases, including those confirmed clinically, epidemiologically, or by laboratory investigation. cholera_case_fatality_rate: title: Cholera case fatality rate - short_unit: '%' unit: '%' - description: WHO calculates case fatality rates based on the numbers of cases - and deaths as reported by national authorities (Ministries of Health). - sources: - - name: World Health Organization (2023) - published_by: World Health Organization - date_accessed: 2023-05-31 + short_unit: '%' + description: |- + WHO calculates case fatality rates based on the numbers of cases and deaths as reported by national authorities (Ministries of Health). cholera_deaths: title: Cholera deaths unit: deaths description: Number of deaths from cholera reported to WHO - sources: - - name: World Health Organization (2023) - published_by: World Health Organization - date_accessed: 2023-05-31 diff --git a/etl/steps/data/grapher/fasttrack/2023-05-31/cholera.py b/etl/steps/data/grapher/fasttrack/2023-05-31/cholera.py index 9282b916edd..740dcc7f72e 100644 --- a/etl/steps/data/grapher/fasttrack/2023-05-31/cholera.py +++ b/etl/steps/data/grapher/fasttrack/2023-05-31/cholera.py @@ -1,19 +1,40 @@ import pandas as pd -from owid import catalog -from etl.helpers import PathFinder, create_dataset +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot -P = PathFinder(__file__) +paths = PathFinder(__file__) def run(dest_dir: str) -> None: # load snapshot - data = pd.read_csv(Snapshot("fasttrack/2023-05-31/cholera.csv").path) + snap = Snapshot("fasttrack/2023-05-31/cholera.csv") - # create empty dataframe and table - tb = catalog.Table(data, short_name=P.short_name) + # load data + tb = snap.read_csv() + + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) # add table, update metadata from *.meta.yml and save - ds = create_dataset(dest_dir, tables=[tb]) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/2023-06-16/guinea_worm.meta.yml b/etl/steps/data/grapher/fasttrack/2023-06-16/guinea_worm.meta.yml index fbd897b7e74..fc674303d0e 100644 --- a/etl/steps/data/grapher/fasttrack/2023-06-16/guinea_worm.meta.yml +++ b/etl/steps/data/grapher/fasttrack/2023-06-16/guinea_worm.meta.yml @@ -1,15 +1,8 @@ dataset: - namespace: fasttrack - version: '2023-06-16' - short_name: guinea_worm title: Guinea Worm Cases - Carter Center (2023) description: The number of cases of guinea worm disease worldwide since 1989 - sources: - - name: The Carter Center (2023) - published_by: The Carter Center (2023) - publication_year: 2023 - date_accessed: 2023-06-16 - url: https://www.cartercenter.org/resources/pdfs/news/health_publications/guinea_worm/guinea-worm-cases-by-year-from-1989.pdf + licenses: + - {} tables: guinea_worm: variables: @@ -17,9 +10,3 @@ tables: title: Guinea Worm Cases unit: cases description: The number of human cases of guinea worm disease. - sources: - - name: The Carter Center (2023) - published_by: The Carter Center (2023) - publication_year: 2023 - date_accessed: 2023-06-16 - url: https://www.cartercenter.org/resources/pdfs/news/health_publications/guinea_worm/guinea-worm-cases-by-year-from-1989.pdf diff --git a/etl/steps/data/grapher/fasttrack/2023-06-16/guinea_worm.py b/etl/steps/data/grapher/fasttrack/2023-06-16/guinea_worm.py index 7d0e4f07318..d247a84999a 100644 --- a/etl/steps/data/grapher/fasttrack/2023-06-16/guinea_worm.py +++ b/etl/steps/data/grapher/fasttrack/2023-06-16/guinea_worm.py @@ -1,19 +1,40 @@ import pandas as pd -from owid import catalog -from etl.helpers import PathFinder, create_dataset +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot -P = PathFinder(__file__) +paths = PathFinder(__file__) def run(dest_dir: str) -> None: # load snapshot - data = pd.read_csv(Snapshot("fasttrack/2023-06-16/guinea_worm.csv").path) + snap = Snapshot("fasttrack/2023-06-16/guinea_worm.csv") - # create empty dataframe and table - tb = catalog.Table(data, short_name=P.short_name) + # load data + tb = snap.read_csv() + + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) # add table, update metadata from *.meta.yml and save - ds = create_dataset(dest_dir, tables=[tb.set_index(["country", "year"])]) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/2023-06-19/world_population_comparison.meta.yml b/etl/steps/data/grapher/fasttrack/2023-06-19/world_population_comparison.meta.yml index 5802a16bd35..60c67e53309 100644 --- a/etl/steps/data/grapher/fasttrack/2023-06-19/world_population_comparison.meta.yml +++ b/etl/steps/data/grapher/fasttrack/2023-06-19/world_population_comparison.meta.yml @@ -1,7 +1,24 @@ dataset: - sources: - - name: Multiple sources compiled by Our World in Data (2019) - published_by: Multiple sources compiled by Our World in Data (2019) + title: Historical world population comparison (various sources) + description: |- + Among others these are the original source: + + McEvedy, Colin and Richard Jones, 1978, “Atlas of World Population History,” Facts on File, New York, pp. 342-351. + + Biraben, Jean-Noel, 1980, An Essay Concerning Mankind’s Evolution, Population, Selected Papers, December, table 2. + + Durand, John D., 1974, “Historical Estimates of World Population: An Evaluation,” University of Pennsylvania, Population Center, Analytical and Technical Reports, Number 10, table 2. + + Haub, Carl, 1995, “How Many People Have Ever Lived on Earth?” Population Today, February, p. 5. + + Thomlinson, Ralph, 1975, “Demographic Problems, Controversy Over Population Control,” Second Edition, Table 1. + + United Nations, 1999, The World at Six Billion, Table 1, “World Population From” Year 0 to Stabilization, p. 5, + U.S. Census Bureau (USCB), 2012, Total Midyear Population for the World: 1950-2050. + + Michael Kremer (1993) “Population Growth and Technological Change: One Million B.C. to 1990”, Quarterly Journal of Economics., August 1993, pp.681-716. + licenses: + - {} tables: world_population_comparison: variables: diff --git a/etl/steps/data/grapher/fasttrack/2023-06-19/world_population_comparison.py b/etl/steps/data/grapher/fasttrack/2023-06-19/world_population_comparison.py index 83d3987374f..de557839e00 100644 --- a/etl/steps/data/grapher/fasttrack/2023-06-19/world_population_comparison.py +++ b/etl/steps/data/grapher/fasttrack/2023-06-19/world_population_comparison.py @@ -1,10 +1,9 @@ import pandas as pd -from owid import catalog -from etl.helpers import PathFinder, create_dataset +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot -P = PathFinder(__file__) +paths = PathFinder(__file__) def run(dest_dir: str) -> None: @@ -12,11 +11,30 @@ def run(dest_dir: str) -> None: snap = Snapshot("fasttrack/2023-06-19/world_population_comparison.csv") # load data - data = pd.read_csv(snap.path) + tb = snap.read_csv() - # create empty dataframe and table - tb = catalog.Table(data, short_name=P.short_name) + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) # add table, update metadata from *.meta.yml and save ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/2023-08-07/pain_hours_days_hen_systems.meta.yml b/etl/steps/data/grapher/fasttrack/2023-08-07/pain_hours_days_hen_systems.meta.yml index 69e507508a6..42c4bfa247b 100644 --- a/etl/steps/data/grapher/fasttrack/2023-08-07/pain_hours_days_hen_systems.meta.yml +++ b/etl/steps/data/grapher/fasttrack/2023-08-07/pain_hours_days_hen_systems.meta.yml @@ -1,119 +1,68 @@ dataset: - sources: - - name: Welfare Footprint based on Schuck-Paim and Alonso (2021) - published_by: Welfare Footprint - publication_year: 2021 - url: https://welfarefootprint.org/research-projects/laying-hens/ + title: Pain hours and days of hen systems (Welfare Footprint) + description: '' + licenses: + - {} tables: pain_hours_days_hen_systems: variables: total_hours_in_pain: title: total_hours_in_pain - short_unit: hours unit: hours - description: The total number of hours an average hen will spend in pain, regardless of the intensity. + short_unit: hours display: numDecimalPlaces: 0.0 - sources: - - name: Welfare Footprint based on Schuck-Paim and Alonso (2021) - published_by: Welfare Footprint - publication_year: 2021 - url: https://welfarefootprint.org/research-projects/laying-hens/ + description: The total number of hours an average hen will spend in pain, regardless of the intensity. excrutiating_hours_in_pain: title: excrutiating_hours_in_pain - short_unit: hours unit: hours - description: The number of hours an average hen will spend in excrutiating pain. + short_unit: hours display: numDecimalPlaces: 2.0 - sources: - - name: Welfare Footprint based on Schuck-Paim and Alonso (2021) - published_by: Welfare Footprint - publication_year: 2021 - url: https://welfarefootprint.org/research-projects/laying-hens/ + description: The number of hours an average hen will spend in excrutiating pain. disabling_pain_in_hours: title: disabling_pain_in_hours - short_unit: hours unit: hours - description: The number of hours an average hen will spend in disabling pain. + short_unit: hours display: numDecimalPlaces: 0.0 - sources: - - name: Welfare Footprint based on Schuck-Paim and Alonso (2021) - published_by: Welfare Footprint - publication_year: 2021 - url: https://welfarefootprint.org/research-projects/laying-hens/ + description: The number of hours an average hen will spend in disabling pain. hurtful_pain_in_hours: title: hurtful_pain_in_hours - short_unit: hours unit: hours - description: The number of hours an average hen will spend in hurtful pain. + short_unit: hours display: numDecimalPlaces: 0.0 - sources: - - name: Welfare Footprint based on Schuck-Paim and Alonso (2021) - published_by: Welfare Footprint - publication_year: 2021 - url: https://welfarefootprint.org/research-projects/laying-hens/ + description: The number of hours an average hen will spend in hurtful pain. annoying_pain_in_hours: title: annoying_pain_in_hours - short_unit: hours unit: hours - description: The number of hours an average hen will spend in annoying pain. + short_unit: hours display: numDecimalPlaces: 0.0 - sources: - - name: Welfare Footprint based on Schuck-Paim and Alonso (2021) - published_by: Welfare Footprint - publication_year: 2021 - url: https://welfarefootprint.org/research-projects/laying-hens/ + description: The number of hours an average hen will spend in annoying pain. total_days_in_pain: title: total_days_in_pain - short_unit: days unit: days + short_unit: days description: The total number of days an average hen will spend in pain, regardless of the intensity. - sources: - - name: Welfare Footprint based on Schuck-Paim and Alonso (2021) - published_by: Welfare Footprint - publication_year: 2021 - url: https://welfarefootprint.org/research-projects/laying-hens/ excrutating_pain_in_days: title: excrutating_pain_in_days - short_unit: days unit: days + short_unit: days description: The number of days an average hen will spend in excrutiating pain. - sources: - - name: Welfare Footprint based on Schuck-Paim and Alonso (2021) - published_by: Welfare Footprint - publication_year: 2021 - url: https://welfarefootprint.org/research-projects/laying-hens/ disabling_pain_in_days: title: disabling_pain_in_days - short_unit: days unit: days + short_unit: days description: The number of days an average hen will spend in disabling pain. - sources: - - name: Welfare Footprint based on Schuck-Paim and Alonso (2021) - published_by: Welfare Footprint - publication_year: 2021 - url: https://welfarefootprint.org/research-projects/laying-hens/ hurtful_pain_in_days: title: hurtful_pain_in_days - short_unit: days unit: days + short_unit: days description: The number of days an average hen will spend in hurtful pain. - sources: - - name: Welfare Footprint based on Schuck-Paim and Alonso (2021) - published_by: Welfare Footprint - publication_year: 2021 - url: https://welfarefootprint.org/research-projects/laying-hens/ annoying_pain_in_days: title: annoying_pain_in_days - short_unit: days unit: days + short_unit: days description: The number of days an average hen will spend in annoying pain. - sources: - - name: Welfare Footprint based on Schuck-Paim and Alonso (2021) - published_by: Welfare Footprint - publication_year: 2021 - url: https://welfarefootprint.org/research-projects/laying-hens/ diff --git a/etl/steps/data/grapher/fasttrack/2023-08-07/pain_hours_days_hen_systems.py b/etl/steps/data/grapher/fasttrack/2023-08-07/pain_hours_days_hen_systems.py index a8b5904420f..7f2af7320fd 100644 --- a/etl/steps/data/grapher/fasttrack/2023-08-07/pain_hours_days_hen_systems.py +++ b/etl/steps/data/grapher/fasttrack/2023-08-07/pain_hours_days_hen_systems.py @@ -1,10 +1,9 @@ import pandas as pd -from owid import catalog -from etl.helpers import PathFinder, create_dataset +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot -P = PathFinder(__file__) +paths = PathFinder(__file__) def run(dest_dir: str) -> None: @@ -12,11 +11,30 @@ def run(dest_dir: str) -> None: snap = Snapshot("fasttrack/2023-08-07/pain_hours_days_hen_systems.csv") # load data - data = pd.read_csv(snap.path) + tb = snap.read_csv() - # create empty dataframe and table - tb = catalog.Table(data, short_name=P.short_name) + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) # add table, update metadata from *.meta.yml and save - ds = create_dataset(dest_dir, tables=[tb.set_index(["country", "year"])], default_metadata=snap.metadata) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/2023-08-21/survey_livestock_oklahoma.meta.yml b/etl/steps/data/grapher/fasttrack/2023-08-21/survey_livestock_oklahoma.meta.yml index f9e0daeeaec..e24c3af8a4a 100644 --- a/etl/steps/data/grapher/fasttrack/2023-08-21/survey_livestock_oklahoma.meta.yml +++ b/etl/steps/data/grapher/fasttrack/2023-08-21/survey_livestock_oklahoma.meta.yml @@ -2,54 +2,35 @@ dataset: title: Survey attitudes to livestock farming (Oklahoma University) description: '' licenses: - - {} - sources: - - name: Food Demand Survey, Oklahoma State University - url: https://web.archive.org/web/20190806000018/http://agecon.okstate.edu/files/january%202018.pdf - publication_year: '2018' - published_by: Oklahoma State University, Department of Agricultural Economics + - {} tables: survey_livestock_oklahoma: variables: strongly_agree: - title: strongly_agree + title: Strongly agree unit: '' short_unit: '%' - display: - name: Strongly agree agree: - title: agree + title: Agree unit: '' short_unit: '%' - display: - name: Agree somewhat_agree: - title: somewhat_agree + title: Somewhat agree unit: '' short_unit: '%' - display: - name: Somewhat agree no_opinion: - title: no_opinion + title: No opinion unit: '' short_unit: '%' - display: - name: No opinion somewhat_disagree: - title: somewhat_disagree + title: Somewhat disagree unit: '' short_unit: '%' - display: - name: Somewhat disagree disagree: - title: disagree + title: Disagree unit: '' short_unit: '%' - display: - name: Disagree strongly_disagree: - title: strongly_disagree + title: Strongly disagree unit: '' short_unit: '%' - display: - name: Strongly disagree diff --git a/etl/steps/data/grapher/fasttrack/2023-08-21/survey_livestock_oklahoma.py b/etl/steps/data/grapher/fasttrack/2023-08-21/survey_livestock_oklahoma.py index a958614e9b8..1723efe0e28 100644 --- a/etl/steps/data/grapher/fasttrack/2023-08-21/survey_livestock_oklahoma.py +++ b/etl/steps/data/grapher/fasttrack/2023-08-21/survey_livestock_oklahoma.py @@ -1,10 +1,9 @@ import pandas as pd -from owid import catalog -from etl.helpers import PathFinder, create_dataset +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot -P = PathFinder(__file__) +paths = PathFinder(__file__) def run(dest_dir: str) -> None: @@ -12,11 +11,30 @@ def run(dest_dir: str) -> None: snap = Snapshot("fasttrack/2023-08-21/survey_livestock_oklahoma.csv") # load data - data = pd.read_csv(snap.path) + tb = snap.read_csv() - # create empty dataframe and table - tb = catalog.Table(data, short_name=P.short_name) + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) # add table, update metadata from *.meta.yml and save - ds = create_dataset(dest_dir, tables=[tb.set_index(["country", "year"])], default_metadata=snap.metadata) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/2023-09-29/un_space_objects.meta.yml b/etl/steps/data/grapher/fasttrack/2023-09-29/un_space_objects.meta.yml index 8fe8448f45e..fef74cf4a55 100644 --- a/etl/steps/data/grapher/fasttrack/2023-09-29/un_space_objects.meta.yml +++ b/etl/steps/data/grapher/fasttrack/2023-09-29/un_space_objects.meta.yml @@ -1,34 +1,17 @@ dataset: title: Online Index of Objects Launched into Outer Space (UN, 2023-09-29) - description: >- - This data is compiled from the Online Index of Objects Launched into Outer Space, maintained by the United Nations Office - for Outer Space Affairs. + description: |- + This data is compiled from the Online Index of Objects Launched into Outer Space, maintained by the United Nations Office for Outer Space Affairs. + Since 1962, the United Nations has maintained a Register of Objects Launched into Outer Space. Originally established as a mechanism to aid the United Nations Committee on the Peaceful Uses of Outer Space in its discussions on the political, legal, and technical issues concerning outer space, the evolution of international space law resulted in space object registration becoming a means of identifying which States' bear international responsibility and liability for space objects. - Since 1962, the United Nations has maintained a Register of Objects Launched into Outer Space. Originally established - as a mechanism to aid the United Nations Committee on the Peaceful Uses of Outer Space in its discussions on the political, - legal, and technical issues concerning outer space, the evolution of international space law resulted in space object - registration becoming a means of identifying which States' bear international responsibility and liability for space objects. - - - The source indicates that around 87% of all satellites, probes, landers, crewed spacecraft, and space station flight elements - launched into Earth orbit or beyond have been registered with the Secretary-General. - + The source indicates that around 87% of all satellites, probes, landers, crewed spacecraft, and space station flight elements launched into Earth orbit or beyond have been registered with the Secretary-General. In the data shown on our charts: - - when an object is launched by a country on behalf of another one, it is attributed to the latter; - - - when a launch is made jointly by several countries, it is recorded in each of these countries' time series, but only - once in the 'World' series. + - when a launch is made jointly by several countries, it is recorded in each of these countries' time series, but only once in the 'World' series. licenses: - {} - sources: - - name: United Nations Office for Outer Space Affairs, Online Index of Objects Launched into Outer Space (2023) - url: https://www.unoosa.org/oosa/osoindex/search-ng.jspx - date_accessed: '2023-09-29' - publication_year: '2023' - published_by: Online Index of Objects Launched into Outer Space (2023) tables: un_space_objects: variables: diff --git a/etl/steps/data/grapher/fasttrack/2023-09-29/un_space_objects.py b/etl/steps/data/grapher/fasttrack/2023-09-29/un_space_objects.py index 908f5a6e699..52ea7d8ffb2 100644 --- a/etl/steps/data/grapher/fasttrack/2023-09-29/un_space_objects.py +++ b/etl/steps/data/grapher/fasttrack/2023-09-29/un_space_objects.py @@ -1,3 +1,5 @@ +import pandas as pd + from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot @@ -11,8 +13,20 @@ def run(dest_dir: str) -> None: # load data tb = snap.read_csv() + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) + # add table, update metadata from *.meta.yml and save - ds = create_dataset(dest_dir, tables=[tb.set_index(["country", "year"])], default_metadata=snap.metadata) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) # override metadata if necessary meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") @@ -20,3 +34,7 @@ def run(dest_dir: str) -> None: ds.update_metadata(meta_path) ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/2023-10-05/great_pacific_garbage_lebreton.meta.yml b/etl/steps/data/grapher/fasttrack/2023-10-05/great_pacific_garbage_lebreton.meta.yml index 3b6733fc4fb..c46186cfae7 100644 --- a/etl/steps/data/grapher/fasttrack/2023-10-05/great_pacific_garbage_lebreton.meta.yml +++ b/etl/steps/data/grapher/fasttrack/2023-10-05/great_pacific_garbage_lebreton.meta.yml @@ -3,27 +3,21 @@ dataset: description: '' licenses: - {} - sources: - - name: Plastics in Great Pacific Garbage Patch (Lebreton et al. 2022) - url: https://www.nature.com/articles/s41598-022-16529-0 - publication_year: '2022' - published_by: Lebreton et al. (2022). Industrialised fishing nations largely contribute to floating plastic pollution - in the North Pacific subtropical gyre. Nature Scientific Reports. tables: great_pacific_garbage_lebreton: variables: share_number_plastic_items: title: share_number_plastic_items - description: The share of hard plastic items, larger than 5 centimetres, found in the Great Pacific Garbage Patch. unit: '%' short_unit: '%' display: numDecimalPlaces: 0 + description: The share of hard plastic items, larger than 5 centimetres, found in the Great Pacific Garbage Patch. share_mass_plastic_items: title: share_mass_plastic_items - description: The share of the mass of hard plastic items, larger than 5 centimetres, found in the Great Pacific Garbage - Patch. unit: '%' short_unit: '%' display: numDecimalPlaces: 2 + description: The share of the mass of hard plastic items, larger than 5 centimetres, found in the Great Pacific Garbage + Patch. diff --git a/etl/steps/data/grapher/fasttrack/2023-10-05/great_pacific_garbage_lebreton.py b/etl/steps/data/grapher/fasttrack/2023-10-05/great_pacific_garbage_lebreton.py index d3d980cab94..fe4e2084af2 100644 --- a/etl/steps/data/grapher/fasttrack/2023-10-05/great_pacific_garbage_lebreton.py +++ b/etl/steps/data/grapher/fasttrack/2023-10-05/great_pacific_garbage_lebreton.py @@ -1,4 +1,6 @@ -from etl.helpers import PathFinder, create_dataset +import pandas as pd + +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot paths = PathFinder(__file__) @@ -11,6 +13,28 @@ def run(dest_dir: str) -> None: # load data tb = snap.read_csv() + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) + # add table, update metadata from *.meta.yml and save - ds = create_dataset(dest_dir, tables=[tb.set_index(["country", "year"])], default_metadata=snap.metadata) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/latest/antimicrobial_usage_livestock.meta.yml b/etl/steps/data/grapher/fasttrack/latest/antimicrobial_usage_livestock.meta.yml index 4651cb6da5d..b10007cd0eb 100644 --- a/etl/steps/data/grapher/fasttrack/latest/antimicrobial_usage_livestock.meta.yml +++ b/etl/steps/data/grapher/fasttrack/latest/antimicrobial_usage_livestock.meta.yml @@ -1,11 +1,9 @@ dataset: - sources: - - name: Mulchandani et al. (2023) - published_by: 'Mulchandani, R., Wang, Y., Gilbert, M., & Van Boeckel, T. P. (2023). Global trends in antimicrobial use - in food-producing animals: 2020 to 2030. PLOS Global Public Health, 3(2), e0001305. https://doi.org/10.1371/journal.pgph.0001305' - publication_year: 2023 - date_accessed: 2023-07-25 - url: https://journals.plos.org/globalpublichealth/article?id=10.1371/journal.pgph.0001305 + title: Antimicrobial usage in livestock + description: |- + This dataset estimates the usage of antimicrobials in livestock (cattle, sheep, chicken, and pigs) by country. Data on antimicrobials comes from government reports, surveillance systems and national surveys. In addition, the authors estimate the biomass of livestock in the country, to adjust for differences in antimicrobial usage by animal size. Biomass data comes from the Food and Agriculture Organization (FAO). 'The PCU represents the total number of animals in a country (alive or slaughtered), multiplied by the average weight of the animal at the time of treatment. Therefore, the PCU is a standardization metric that accounts for differences in animal weight, and number of production cycles per year between countries.' Therefore, mg/PCU refers to the usage of antimicrobials per animal population-corrected unit. + licenses: + - {} tables: antimicrobial_usage_livestock: variables: diff --git a/etl/steps/data/grapher/fasttrack/latest/antimicrobial_usage_livestock.py b/etl/steps/data/grapher/fasttrack/latest/antimicrobial_usage_livestock.py index 5d9877bfb65..390bab01951 100644 --- a/etl/steps/data/grapher/fasttrack/latest/antimicrobial_usage_livestock.py +++ b/etl/steps/data/grapher/fasttrack/latest/antimicrobial_usage_livestock.py @@ -1,10 +1,9 @@ import pandas as pd -from owid import catalog -from etl.helpers import PathFinder, create_dataset +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot -P = PathFinder(__file__) +paths = PathFinder(__file__) def run(dest_dir: str) -> None: @@ -12,11 +11,30 @@ def run(dest_dir: str) -> None: snap = Snapshot("fasttrack/latest/antimicrobial_usage_livestock.csv") # load data - data = pd.read_csv(snap.path) + tb = snap.read_csv() - # create empty dataframe and table - tb = catalog.Table(data, short_name=P.short_name) + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) # add table, update metadata from *.meta.yml and save ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/latest/cumulative_lives_saved_vaccination_shattock.py b/etl/steps/data/grapher/fasttrack/latest/cumulative_lives_saved_vaccination_shattock.py index d3b822d1f10..ed50f459b1a 100644 --- a/etl/steps/data/grapher/fasttrack/latest/cumulative_lives_saved_vaccination_shattock.py +++ b/etl/steps/data/grapher/fasttrack/latest/cumulative_lives_saved_vaccination_shattock.py @@ -1,3 +1,5 @@ +import pandas as pd + from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot @@ -11,8 +13,20 @@ def run(dest_dir: str) -> None: # load data tb = snap.read_csv() + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) + # add table, update metadata from *.meta.yml and save - ds = create_dataset(dest_dir, tables=[tb.set_index(["country", "year"])], default_metadata=snap.metadata) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) # override metadata if necessary meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") @@ -20,3 +34,7 @@ def run(dest_dir: str) -> None: ds.update_metadata(meta_path) ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/latest/democracy_freedom_house.meta.yml b/etl/steps/data/grapher/fasttrack/latest/democracy_freedom_house.meta.yml index 046439e36eb..6e9511619ea 100644 --- a/etl/steps/data/grapher/fasttrack/latest/democracy_freedom_house.meta.yml +++ b/etl/steps/data/grapher/fasttrack/latest/democracy_freedom_house.meta.yml @@ -1,171 +1,141 @@ dataset: title: Democracy - Freedom House (2023) - description: >- + description: |- This dataset provides information on political regimes, using data from Freedom House's Freedom in the World (2023). - You can read a description of the data in this post: https://ourworldindata.org/democracies-measurement - You can download the code and complete dataset, including supplementary variables, from GitHub: https://github.com/owid/notebooks/tree/main/BastianHerre/democracy licenses: - - {} - sources: - - name: Freedom House (2023) - url: https://freedomhouse.org/report/freedom-world - date_accessed: '2023-09-05' - published_by: Freedom House (2023). Freedom in the World. + - {} tables: democracy_freedom_house: variables: regime_fh: title: regime_fh - description: >- - The variable identifies the political regime of a country using the classification by Freedom House's Freedom in - the World. It distinguishes between free countries (score 2), partly free countries (score 1), and not free countries - (score 0). - + unit: '' + display: + name: Political regime + description: |- + The variable identifies the political regime of a country using the classification by Freedom House's Freedom in the World. It distinguishes between free countries (score 2), partly free countries (score 1), and not free countries (score 0). It matches Freedom House's variable scorest. - - Free countries are understood as countries in which citizens have many political rights (free and fair elections, - political pluralism and participation, functioning government) and civil liberties (freedoms of expression and association, - rule of law, personal autonomy). - + Free countries are understood as countries in which citizens have many political rights (free and fair elections, political pluralism and participation, functioning government) and civil liberties (freedoms of expression and association, rule of law, personal autonomy). Partly free country are countries in which citizens have some political rights and civil liberties. - Not free country are countries in which citizens have few political rights and civil liberties. - unit: '' - display: - name: Political regime polrights_fh: title: Political rights - description: >- - The variable identifies the extent of free and fair elections, political pluralism and participation, and a functioning - government, based on Freedom House's Freedom in the World. - + unit: '' + description: |- + The variable identifies the extent of free and fair elections, political pluralism and participation, and a functioning government, based on Freedom House's Freedom in the World. It matches Freedom House's variable scorepr. - Lower ratings indicate more rights. - unit: '' civlibs_fh: title: Civil liberties - description: >- - The variable identifies the extent of freedom of expression and association, the rule of law, and personal autonomy, - based on Freedom House's Freedom in the World. - + unit: '' + description: |- + The variable identifies the extent of freedom of expression and association, the rule of law, and personal autonomy, based on Freedom House's Freedom in the World. It matches Freedom House's variable scorecl. - Lower ratings indicate more liberties. - unit: '' electdem_fh: title: Political regime - description: >- - The variable identifies the political regime of a country using the second classification by Freedom House's Freedom - in the World. It identifies whether a country is an electoral democracy (score 1) or not (score 0). - - - Electoral democracies are understood as political systems in which citizens have the right to choose chief executive - and legislature in broadly free and fair elections and have substantial other political rights and civil liberties. unit: '' + description: |- + The variable identifies the political regime of a country using the second classification by Freedom House's Freedom in the World. It identifies whether a country is an electoral democracy (score 1) or not (score 0). + + Electoral democracies are understood as political systems in which citizens have the right to choose chief executive and legislature in broadly free and fair elections and have substantial other political rights and civil liberties. electprocess_fh: title: Democratic electoral institutions - description: >- - The variable identifies the extent to which the chief executive and legislature are chosen in free and fair elections - under universal suffrage, based on Freedom House's Freedom in the World. - + unit: '' + description: |- + The variable identifies the extent to which the chief executive and legislature are chosen in free and fair elections under universal suffrage, based on Freedom House's Freedom in the World. Higher scores indicate a freer electoral process. - unit: '' polrights_score_fh: title: polrights_score_fh - description: >- - The variable identifies the fine-grained extent of free and fair elections, political pluralism and participation, - and a functioning government, based on Freedom House's Freedom in the World. - - - Higher scores indicate more rights. unit: '' display: name: Political rights + description: |- + The variable identifies the fine-grained extent of free and fair elections, political pluralism and participation, and a functioning government, based on Freedom House's Freedom in the World. + + Higher scores indicate more rights. civlibs_score_fh: title: civlibs_score_fh - description: >- - The variable identifies the fine-grained extent of freedom of expression and association, the rule of law, and personal - autonomy, based on Freedom House's Freedom in the World. - - - Higher scores indicate more liberties. unit: '' display: name: Civil liberties + description: |- + The variable identifies the fine-grained extent of freedom of expression and association, the rule of law, and personal autonomy, based on Freedom House's Freedom in the World. + + Higher scores indicate more liberties. country_fh: title: country_fh - description: The variable identifies whether Freedom House consider the entity a country or territory. unit: '' + description: The variable identifies whether Freedom House consider the entity a country or territory. number_notfree_fh: title: number_notfree_fh - description: The variable identifies the number of not-free countries based on Freedom House's Freedom in the World. unit: '' + description: The variable identifies the number of not-free countries based on Freedom House's Freedom in the World. number_partlyfree_fh: title: number_partlyfree_fh - description: The variable identifies the number of partly-free countries based on Freedom House's Freedom in the World. unit: '' + description: The variable identifies the number of partly-free countries based on Freedom House's Freedom in the World. number_free_fh: title: number_free_fh - description: The variable identifies the number of free countries based on Freedom House's Freedom in the World. unit: '' + description: The variable identifies the number of free countries based on Freedom House's Freedom in the World. number_nonelectdem_fh: title: number_nonelectdem_fh - description: The variable identifies the number of non-democracies based on Freedom House's Freedom in the World. unit: '' + description: The variable identifies the number of non-democracies based on Freedom House's Freedom in the World. number_electdem_fh: title: number_electdem_fh - description: The variable identifies the number of electoral democracies based on Freedom House's Freedom in the World. unit: '' + description: The variable identifies the number of electoral democracies based on Freedom House's Freedom in the World. pop_notfree_fh: title: pop_notfree_fh - description: The variable identifies the number of people living in not free countries, based on Freedom House's Freedom - in the World. unit: '' + description: |- + The variable identifies the number of people living in not free countries, based on Freedom House's Freedom in the World. pop_partlyfree_fh: title: pop_partlyfree_fh - description: The variable identifies the number of people living in partly free countries, based on Freedom House's - Freedom in the World. unit: '' + description: |- + The variable identifies the number of people living in partly free countries, based on Freedom House's Freedom in the World. pop_free_fh: title: pop_free_fh + unit: '' description: The variable identifies the number of people living in free countries, based on Freedom House's Freedom in the World. - unit: '' pop_missreg_fh: title: pop_missreg_fh - description: The variable identifies the number of people living in countries without regime data, based on Freedom - House's Freedom in the World. unit: '' + description: |- + The variable identifies the number of people living in countries without regime data, based on Freedom House's Freedom in the World. pop_nonelectdem_fh: title: pop_nonelectdem_fh + unit: '' description: The variable identifies the number of people living in non-democracies, based on Freedom House's Freedom in the World. - unit: '' pop_electdem_fh: title: pop_electdem_fh - description: The variable identifies the number of people living in electoral democracies, based on Freedom House's - Freedom in the World. unit: '' + description: |- + The variable identifies the number of people living in electoral democracies, based on Freedom House's Freedom in the World. pop_missdem_fh: title: pop_missdem_fh - description: The variable identifies the number of people living in countries without democracy data, based on Freedom - House's Freedom in the World. unit: '' + description: |- + The variable identifies the number of people living in countries without democracy data, based on Freedom House's Freedom in the World. region: title: region - description: The variable identifies the region a country or territory is in. unit: '' + description: The variable identifies the region a country or territory is in. diff --git a/etl/steps/data/grapher/fasttrack/latest/democracy_freedom_house.py b/etl/steps/data/grapher/fasttrack/latest/democracy_freedom_house.py index d72eedbccc5..eff617c8ff4 100644 --- a/etl/steps/data/grapher/fasttrack/latest/democracy_freedom_house.py +++ b/etl/steps/data/grapher/fasttrack/latest/democracy_freedom_house.py @@ -1,7 +1,6 @@ import pandas as pd -from owid import catalog -from etl.helpers import PathFinder, create_dataset +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot paths = PathFinder(__file__) @@ -12,11 +11,30 @@ def run(dest_dir: str) -> None: snap = Snapshot("fasttrack/latest/democracy_freedom_house.csv") # load data - data = pd.read_csv(snap.path) + tb = snap.read_csv() - # create empty dataframe and table - tb = catalog.Table(data, short_name=paths.short_name) + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) # add table, update metadata from *.meta.yml and save - ds = create_dataset(dest_dir, tables=[tb.set_index(["country", "year"])], default_metadata=snap.metadata) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/latest/gbd_2019_mental_health_country_coverage.meta.yml b/etl/steps/data/grapher/fasttrack/latest/gbd_2019_mental_health_country_coverage.meta.yml index 384f41c717e..777a5a9a304 100644 --- a/etl/steps/data/grapher/fasttrack/latest/gbd_2019_mental_health_country_coverage.meta.yml +++ b/etl/steps/data/grapher/fasttrack/latest/gbd_2019_mental_health_country_coverage.meta.yml @@ -1,25 +1,11 @@ dataset: - namespace: fasttrack - version: latest - short_name: gbd_2019_mental_health_country_coverage title: Countries with mental health data in GBD 2019 description: |- Dataset showing the number of countries with primary data on the prevalence of mental illnesses. These were found after a systematic review, grey literature search and expert consultation, to identify studies with data on the prevalence of each mental illness. 'The GBD inclusion criteria stipulated that: (1) the diagnostic criteria must be from 1980 onward; (2) “caseness” must be based on clinical threshold as established by the DSM, ICD, Chinese Classification of Mental Disorders (CCMD), or diagnosed by a clinician using established tools; (3) sufficient information must be provided on study method and sample characteristics to assess the quality of the study; and (4) study samples must be representative of the general population (i.e., case studies, veterans, or refugee samples were excluded). No limitation was set on the language of publication.' - sources: - - name: IHME GBD (2019) - published_by: 'Vos, T., Lim, S. S., Abbafati, C., Abbas, K. M., Abbasi, M., Abbasifard, - M., Abbasi-Kangevari, M., Abbastabar, H., Abd-Allah, F., Abdelalim, A., Abdollahi, - M., Abdollahpour, I., Abolhassani, H., Aboyans, V., Abrams, E. M., Abreu, L. - G., Abrigo, M. R. M., Abu-Raddad, L. J., Abushouk, A. I., … Murray, C. J. L. - (2020). Global burden of 369 diseases and injuries in 204 countries and territories, - 1990–2019: A systematic analysis for the Global Burden of Disease Study 2019. - The Lancet, 396(10258), 1204–1222.' - description: Country-level prevalence data - publication_year: 2020 - date_accessed: 2023-05-05 - url: https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(20)30925-9/fulltext + licenses: + - url: https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(20)30925-9/fulltext tables: gbd_2019_mental_health_country_coverage: variables: diff --git a/etl/steps/data/grapher/fasttrack/latest/gbd_2019_mental_health_country_coverage.py b/etl/steps/data/grapher/fasttrack/latest/gbd_2019_mental_health_country_coverage.py index f50cd5e8481..b43cc1b4135 100644 --- a/etl/steps/data/grapher/fasttrack/latest/gbd_2019_mental_health_country_coverage.py +++ b/etl/steps/data/grapher/fasttrack/latest/gbd_2019_mental_health_country_coverage.py @@ -1,19 +1,40 @@ import pandas as pd -from owid import catalog -from etl.helpers import PathFinder, create_dataset +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot -P = PathFinder(__file__) +paths = PathFinder(__file__) def run(dest_dir: str) -> None: # load snapshot - data = pd.read_csv(Snapshot("fasttrack/latest/gbd_2019_mental_health_country_coverage.csv").path) + snap = Snapshot("fasttrack/latest/gbd_2019_mental_health_country_coverage.csv") - # create empty dataframe and table - tb = catalog.Table(data, short_name=P.short_name) + # load data + tb = snap.read_csv() + + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) # add table, update metadata from *.meta.yml and save - ds = create_dataset(dest_dir, tables=[tb]) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/latest/global_maternal_offspring_loss.meta.yml b/etl/steps/data/grapher/fasttrack/latest/global_maternal_offspring_loss.meta.yml index 60e45e07800..10c7516ebcb 100644 --- a/etl/steps/data/grapher/fasttrack/latest/global_maternal_offspring_loss.meta.yml +++ b/etl/steps/data/grapher/fasttrack/latest/global_maternal_offspring_loss.meta.yml @@ -1,28 +1,17 @@ dataset: title: Global maternal offspring loss - Smith-Greenaway et al. 2021 - description: This dataset shows survey data and estimates of maternal offspring loss across countries. This includes mothers - who have lost an infant, child under 5 years old, or offspring. These are given as a rate per 1000 women in the age group. - Underlying data comes from large-scale surveys (such as the Demographic and Health Surveys and Multiple Indicator Cluster - Surveys) conducted in many low- and middle-income countries. For countries lacking data, these are estimated using an - indirect approach that combines formal kinship models and life-table methods in an additional 81 countries. + description: |- + This dataset shows survey data and estimates of maternal offspring loss across countries. This includes mothers who have lost an infant, child under 5 years old, or offspring. These are given as a rate per 1000 women in the age group. Underlying data comes from large-scale surveys (such as the Demographic and Health Surveys and Multiple Indicator Cluster Surveys) conducted in many low- and middle-income countries. For countries lacking data, these are estimated using an indirect approach that combines formal kinship models and life-table methods in an additional 81 countries. Citation: Smith-Greenaway, E., Alburez-Gutierrez, D., Trinitapoli, J., & Zagheni, E. (2021). Global burden of maternal bereavement: Indicators of the cumulative prevalence of child loss. BMJ Global Health, 6(4), e004837. https://doi.org/10.1136/bmjgh-2020-004837 licenses: - - {} - sources: - - name: Smith-Greenaway et al. (2021) - url: https://gh.bmj.com/content/6/4/e004837.abstract - date_accessed: '2023-09-05' - publication_year: '2021' - published_by: 'Global burden of maternal bereavement: indicators of the cumulative prevalence of child loss. (2021) Emily - Smith-Greenaway, Diego Alburez-Gutierrez, Jenny Trinitapoli, Emilio Zagheni.' + - {} tables: global_maternal_offspring_loss: variables: survey_source: title: Survey source - description: The survey used for underlying data – either the Demographic and Health Surveys (DHS) or Multiple Indicator - Cluster Surveys (MICS). Data was estimated for countries without a survey source, using formal kinship-models and - life tables. unit: '' + description: |- + The survey used for underlying data – either the Demographic and Health Surveys (DHS) or Multiple Indicator Cluster Surveys (MICS). Data was estimated for countries without a survey source, using formal kinship-models and life tables. cumulative_maternal_infant_mortality_among_mothers_aged_20_44_as_a_rate_per_1000_mothers_in_the_age_group: title: Cumulative maternal infant mortality among mothers aged 20-44 as a rate per 1000 mothers in the age group unit: per 1000 mothers diff --git a/etl/steps/data/grapher/fasttrack/latest/global_maternal_offspring_loss.py b/etl/steps/data/grapher/fasttrack/latest/global_maternal_offspring_loss.py index 6cd1b71eae7..b0ff839ac7b 100644 --- a/etl/steps/data/grapher/fasttrack/latest/global_maternal_offspring_loss.py +++ b/etl/steps/data/grapher/fasttrack/latest/global_maternal_offspring_loss.py @@ -1,7 +1,6 @@ import pandas as pd -from owid import catalog -from etl.helpers import PathFinder, create_dataset +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot paths = PathFinder(__file__) @@ -12,11 +11,30 @@ def run(dest_dir: str) -> None: snap = Snapshot("fasttrack/latest/global_maternal_offspring_loss.csv") # load data - data = pd.read_csv(snap.path) + tb = snap.read_csv() - # create empty dataframe and table - tb = catalog.Table(data, short_name=paths.short_name) + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) # add table, update metadata from *.meta.yml and save - ds = create_dataset(dest_dir, tables=[tb.set_index(["country", "year"])], default_metadata=snap.metadata) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/latest/historical_france_mortality_cause.meta.yml b/etl/steps/data/grapher/fasttrack/latest/historical_france_mortality_cause.meta.yml index 2af4f53a6b7..f785bb48b59 100644 --- a/etl/steps/data/grapher/fasttrack/latest/historical_france_mortality_cause.meta.yml +++ b/etl/steps/data/grapher/fasttrack/latest/historical_france_mortality_cause.meta.yml @@ -1,6611 +1,6513 @@ dataset: title: Database on causes of death in France from 1925 to 1999 - description: >- - Dataset on mortality rates from each cause of death category in France between 1925 and 1999. The underlying data for - this chart comes from the Institut National d'Études Démographiques, published by Jacques Vallin and France Meslé, and - covers causes of deaths nationally in France between 1925 and 1999. Causes of death were categorized into categories according - to the 9th edition of the International Classification of Diseases (ICD-9) manual. Mortality rates are given for five-year - age bands, as an annual rate out of 100,000 people in that age group. Below are the ICD codes used for each cause category: - All causes = 000*-999*, - + description: |- + Dataset on mortality rates from each cause of death category in France between 1925 and 1999. The underlying data for this chart comes from the Institut National d'Études Démographiques, published by Jacques Vallin and France Meslé, and covers causes of deaths nationally in France between 1925 and 1999. Causes of death were categorized into categories according to the 9th edition of the International Classification of Diseases (ICD-9) manual. Mortality rates are given for five-year age bands, as an annual rate out of 100,000 people in that age group. Below are the ICD codes used for each cause category: All causes = 000*-999*, Infectious and parasitic diseases = 001*-139*, - Neoplasms = 140*-239*, - Endocrine nutritional and metabolic diseases and immunity disorders = 240*-279*, - Diseases of the blood and blood-forming organs = 280*-289*, - Mental disorders = 290*-319*, - Diseases of the nervous system = 320*-359*, - Diseases of the sense organs = 360*-389*, - Diseases of the circulatory system = 390*-459*, - Diseases of the respiratory system = 460*-519*, - Diseases of the digestive system = 520*-579*, - Diseases of the genitourinary system = 580*-629*, - Complications of pregnancy childbirth and the puerperium = 630*-679*, - Diseases of the skin and subcutaneous tissue = 680*-709*, - Diseases of the musculoskeletal system and connective tissue = 710*-739*, - Congenital anomalies = 740*-759*, - Certain conditions originating in the perinatal period = 760*-779*, - Symptoms signs and ill-defined conditions = 780*-799*, - External causes (injury and poisoning) = 800*-999* licenses: - - {} - sources: - - name: Institut National d'Études Démographiques - url: https://www.demographic-research.org/Volumes/Vol36/21/ - date_accessed: '2023-06-23' - publication_year: '2014' - published_by: Jacques Vallin and France Meslé + - {} tables: historical_france_mortality_cause: variables: all_causes_female_age_all_ages: title: All causes_female_age_all ages - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_under_1: title: All causes_female_age_under_1 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_1_4: title: All causes_female_age_1-4 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_5_9: title: All causes_female_age_5-9 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_10_14: title: All causes_female_age_10-14 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_15_19: title: All causes_female_age_15-19 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_20_24: title: All causes_female_age_20-24 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_25_29: title: All causes_female_age_25-29 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_30_34: title: All causes_female_age_30-34 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_35_39: title: All causes_female_age_35-39 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_40_44: title: All causes_female_age_40-44 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_45_49: title: All causes_female_age_45-49 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_50_54: title: All causes_female_age_50-54 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_55_59: title: All causes_female_age_55-59 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_60_64: title: All causes_female_age_60-64 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_65_69: title: All causes_female_age_65-69 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_70_74: title: All causes_female_age_70-74 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_75_79: title: All causes_female_age_75-79 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_80_84: title: All causes_female_age_80-84 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_85_89: title: All causes_female_age_85-89 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_90_94: title: All causes_female_age_90-94 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_95_99: title: All causes_female_age_95-99 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_female_age_100_and_over: title: All causes_female_age_100_and_over - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. infectious_and_parasitic_diseases_female_age_all_ages: title: Infectious and parasitic diseases_female_age_all ages - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_under_1: title: Infectious and parasitic diseases_female_age_under_1 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_1_4: title: Infectious and parasitic diseases_female_age_1-4 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_5_9: title: Infectious and parasitic diseases_female_age_5-9 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_10_14: title: Infectious and parasitic diseases_female_age_10-14 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_15_19: title: Infectious and parasitic diseases_female_age_15-19 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_20_24: title: Infectious and parasitic diseases_female_age_20-24 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_25_29: title: Infectious and parasitic diseases_female_age_25-29 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_30_34: title: Infectious and parasitic diseases_female_age_30-34 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_35_39: title: Infectious and parasitic diseases_female_age_35-39 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_40_44: title: Infectious and parasitic diseases_female_age_40-44 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_45_49: title: Infectious and parasitic diseases_female_age_45-49 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_50_54: title: Infectious and parasitic diseases_female_age_50-54 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_55_59: title: Infectious and parasitic diseases_female_age_55-59 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_60_64: title: Infectious and parasitic diseases_female_age_60-64 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_65_69: title: Infectious and parasitic diseases_female_age_65-69 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_70_74: title: Infectious and parasitic diseases_female_age_70-74 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_75_79: title: Infectious and parasitic diseases_female_age_75-79 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_80_84: title: Infectious and parasitic diseases_female_age_80-84 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_85_89: title: Infectious and parasitic diseases_female_age_85-89 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_90_94: title: Infectious and parasitic diseases_female_age_90-94 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_95_99: title: Infectious and parasitic diseases_female_age_95-99 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_female_age_100_and_over: title: Infectious and parasitic diseases_female_age_100_and_over - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. neoplasms_female_age_all_ages: title: Neoplasms_female_age_all ages + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_under_1: title: Neoplasms_female_age_under_1 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_1_4: title: Neoplasms_female_age_1-4 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_5_9: title: Neoplasms_female_age_5-9 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_10_14: title: Neoplasms_female_age_10-14 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_15_19: title: Neoplasms_female_age_15-19 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_20_24: title: Neoplasms_female_age_20-24 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_25_29: title: Neoplasms_female_age_25-29 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_30_34: title: Neoplasms_female_age_30-34 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_35_39: title: Neoplasms_female_age_35-39 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_40_44: title: Neoplasms_female_age_40-44 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_45_49: title: Neoplasms_female_age_45-49 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_50_54: title: Neoplasms_female_age_50-54 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_55_59: title: Neoplasms_female_age_55-59 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_60_64: title: Neoplasms_female_age_60-64 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_65_69: title: Neoplasms_female_age_65-69 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_70_74: title: Neoplasms_female_age_70-74 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_75_79: title: Neoplasms_female_age_75-79 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_80_84: title: Neoplasms_female_age_80-84 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_85_89: title: Neoplasms_female_age_85-89 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_90_94: title: Neoplasms_female_age_90-94 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_95_99: title: Neoplasms_female_age_95-99 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_female_age_100_and_over: title: Neoplasms_female_age_100_and_over + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_all_ages: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_all ages - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_under_1: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_under_1 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_1_4: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_1-4 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_5_9: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_5-9 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_10_14: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_10-14 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_15_19: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_15-19 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_20_24: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_20-24 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_25_29: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_25-29 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_30_34: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_30-34 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_35_39: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_35-39 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_40_44: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_40-44 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_45_49: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_45-49 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_50_54: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_50-54 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_55_59: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_55-59 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_60_64: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_60-64 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_65_69: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_65-69 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_70_74: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_70-74 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_75_79: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_75-79 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_80_84: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_80-84 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_85_89: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_85-89 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_90_94: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_90-94 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_95_99: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_95-99 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_female_age_100_and_over: title: Endocrine nutritional and metabolic diseases and immunity disorders_female_age_100_and_over - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. diseases_of_the_blood_and_blood_forming_organs_female_age_all_ages: title: Diseases of the blood and blood-forming organs_female_age_all ages - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_under_1: title: Diseases of the blood and blood-forming organs_female_age_under_1 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_1_4: title: Diseases of the blood and blood-forming organs_female_age_1-4 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_5_9: title: Diseases of the blood and blood-forming organs_female_age_5-9 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_10_14: title: Diseases of the blood and blood-forming organs_female_age_10-14 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_15_19: title: Diseases of the blood and blood-forming organs_female_age_15-19 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_20_24: title: Diseases of the blood and blood-forming organs_female_age_20-24 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_25_29: title: Diseases of the blood and blood-forming organs_female_age_25-29 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_30_34: title: Diseases of the blood and blood-forming organs_female_age_30-34 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_35_39: title: Diseases of the blood and blood-forming organs_female_age_35-39 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_40_44: title: Diseases of the blood and blood-forming organs_female_age_40-44 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_45_49: title: Diseases of the blood and blood-forming organs_female_age_45-49 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_50_54: title: Diseases of the blood and blood-forming organs_female_age_50-54 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_55_59: title: Diseases of the blood and blood-forming organs_female_age_55-59 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_60_64: title: Diseases of the blood and blood-forming organs_female_age_60-64 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_65_69: title: Diseases of the blood and blood-forming organs_female_age_65-69 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_70_74: title: Diseases of the blood and blood-forming organs_female_age_70-74 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_75_79: title: Diseases of the blood and blood-forming organs_female_age_75-79 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_80_84: title: Diseases of the blood and blood-forming organs_female_age_80-84 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_85_89: title: Diseases of the blood and blood-forming organs_female_age_85-89 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_90_94: title: Diseases of the blood and blood-forming organs_female_age_90-94 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_95_99: title: Diseases of the blood and blood-forming organs_female_age_95-99 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_female_age_100_and_over: title: Diseases of the blood and blood-forming organs_female_age_100_and_over - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. mental_disorders_female_age_all_ages: title: Mental disorders_female_age_all ages - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_under_1: title: Mental disorders_female_age_under_1 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_1_4: title: Mental disorders_female_age_1-4 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_5_9: title: Mental disorders_female_age_5-9 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_10_14: title: Mental disorders_female_age_10-14 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_15_19: title: Mental disorders_female_age_15-19 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_20_24: title: Mental disorders_female_age_20-24 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_25_29: title: Mental disorders_female_age_25-29 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_30_34: title: Mental disorders_female_age_30-34 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_35_39: title: Mental disorders_female_age_35-39 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_40_44: title: Mental disorders_female_age_40-44 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_45_49: title: Mental disorders_female_age_45-49 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_50_54: title: Mental disorders_female_age_50-54 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_55_59: title: Mental disorders_female_age_55-59 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_60_64: title: Mental disorders_female_age_60-64 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_65_69: title: Mental disorders_female_age_65-69 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_70_74: title: Mental disorders_female_age_70-74 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_75_79: title: Mental disorders_female_age_75-79 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_80_84: title: Mental disorders_female_age_80-84 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_85_89: title: Mental disorders_female_age_85-89 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_90_94: title: Mental disorders_female_age_90-94 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_95_99: title: Mental disorders_female_age_95-99 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_female_age_100_and_over: title: Mental disorders_female_age_100_and_over - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. diseases_of_the_nervous_system_female_age_all_ages: title: Diseases of the nervous system_female_age_all ages - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_under_1: title: Diseases of the nervous system_female_age_under_1 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_1_4: title: Diseases of the nervous system_female_age_1-4 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_5_9: title: Diseases of the nervous system_female_age_5-9 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_10_14: title: Diseases of the nervous system_female_age_10-14 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_15_19: title: Diseases of the nervous system_female_age_15-19 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_20_24: title: Diseases of the nervous system_female_age_20-24 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_25_29: title: Diseases of the nervous system_female_age_25-29 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_30_34: title: Diseases of the nervous system_female_age_30-34 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_35_39: title: Diseases of the nervous system_female_age_35-39 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_40_44: title: Diseases of the nervous system_female_age_40-44 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_45_49: title: Diseases of the nervous system_female_age_45-49 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_50_54: title: Diseases of the nervous system_female_age_50-54 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_55_59: title: Diseases of the nervous system_female_age_55-59 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_60_64: title: Diseases of the nervous system_female_age_60-64 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_65_69: title: Diseases of the nervous system_female_age_65-69 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_70_74: title: Diseases of the nervous system_female_age_70-74 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_75_79: title: Diseases of the nervous system_female_age_75-79 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_80_84: title: Diseases of the nervous system_female_age_80-84 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_85_89: title: Diseases of the nervous system_female_age_85-89 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_90_94: title: Diseases of the nervous system_female_age_90-94 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_95_99: title: Diseases of the nervous system_female_age_95-99 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_female_age_100_and_over: title: Diseases of the nervous system_female_age_100_and_over - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_sense_organs_female_age_all_ages: title: Diseases of the sense organs_female_age_all ages - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_under_1: title: Diseases of the sense organs_female_age_under_1 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_1_4: title: Diseases of the sense organs_female_age_1-4 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_5_9: title: Diseases of the sense organs_female_age_5-9 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_10_14: title: Diseases of the sense organs_female_age_10-14 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_15_19: title: Diseases of the sense organs_female_age_15-19 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_20_24: title: Diseases of the sense organs_female_age_20-24 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_25_29: title: Diseases of the sense organs_female_age_25-29 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_30_34: title: Diseases of the sense organs_female_age_30-34 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_35_39: title: Diseases of the sense organs_female_age_35-39 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_40_44: title: Diseases of the sense organs_female_age_40-44 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_45_49: title: Diseases of the sense organs_female_age_45-49 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_50_54: title: Diseases of the sense organs_female_age_50-54 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_55_59: title: Diseases of the sense organs_female_age_55-59 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_60_64: title: Diseases of the sense organs_female_age_60-64 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_65_69: title: Diseases of the sense organs_female_age_65-69 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_70_74: title: Diseases of the sense organs_female_age_70-74 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_75_79: title: Diseases of the sense organs_female_age_75-79 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_80_84: title: Diseases of the sense organs_female_age_80-84 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_85_89: title: Diseases of the sense organs_female_age_85-89 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_90_94: title: Diseases of the sense organs_female_age_90-94 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_95_99: title: Diseases of the sense organs_female_age_95-99 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_female_age_100_and_over: title: Diseases of the sense organs_female_age_100_and_over - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_circulatory_system_female_age_all_ages: title: Diseases of the circulatory system_female_age_all ages - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_under_1: title: Diseases of the circulatory system_female_age_under_1 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_1_4: title: Diseases of the circulatory system_female_age_1-4 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_5_9: title: Diseases of the circulatory system_female_age_5-9 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_10_14: title: Diseases of the circulatory system_female_age_10-14 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_15_19: title: Diseases of the circulatory system_female_age_15-19 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_20_24: title: Diseases of the circulatory system_female_age_20-24 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_25_29: title: Diseases of the circulatory system_female_age_25-29 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_30_34: title: Diseases of the circulatory system_female_age_30-34 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_35_39: title: Diseases of the circulatory system_female_age_35-39 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_40_44: title: Diseases of the circulatory system_female_age_40-44 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_45_49: title: Diseases of the circulatory system_female_age_45-49 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_50_54: title: Diseases of the circulatory system_female_age_50-54 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_55_59: title: Diseases of the circulatory system_female_age_55-59 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_60_64: title: Diseases of the circulatory system_female_age_60-64 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_65_69: title: Diseases of the circulatory system_female_age_65-69 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_70_74: title: Diseases of the circulatory system_female_age_70-74 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_75_79: title: Diseases of the circulatory system_female_age_75-79 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_80_84: title: Diseases of the circulatory system_female_age_80-84 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_85_89: title: Diseases of the circulatory system_female_age_85-89 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_90_94: title: Diseases of the circulatory system_female_age_90-94 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_95_99: title: Diseases of the circulatory system_female_age_95-99 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_female_age_100_and_over: title: Diseases of the circulatory system_female_age_100_and_over - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_respiratory_system_female_age_all_ages: title: Diseases of the respiratory system_female_age_all ages - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_under_1: title: Diseases of the respiratory system_female_age_under_1 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_1_4: title: Diseases of the respiratory system_female_age_1-4 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_5_9: title: Diseases of the respiratory system_female_age_5-9 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_10_14: title: Diseases of the respiratory system_female_age_10-14 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_15_19: title: Diseases of the respiratory system_female_age_15-19 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_20_24: title: Diseases of the respiratory system_female_age_20-24 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_25_29: title: Diseases of the respiratory system_female_age_25-29 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_30_34: title: Diseases of the respiratory system_female_age_30-34 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_35_39: title: Diseases of the respiratory system_female_age_35-39 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_40_44: title: Diseases of the respiratory system_female_age_40-44 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_45_49: title: Diseases of the respiratory system_female_age_45-49 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_50_54: title: Diseases of the respiratory system_female_age_50-54 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_55_59: title: Diseases of the respiratory system_female_age_55-59 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_60_64: title: Diseases of the respiratory system_female_age_60-64 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_65_69: title: Diseases of the respiratory system_female_age_65-69 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_70_74: title: Diseases of the respiratory system_female_age_70-74 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_75_79: title: Diseases of the respiratory system_female_age_75-79 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_80_84: title: Diseases of the respiratory system_female_age_80-84 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_85_89: title: Diseases of the respiratory system_female_age_85-89 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_90_94: title: Diseases of the respiratory system_female_age_90-94 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_95_99: title: Diseases of the respiratory system_female_age_95-99 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_female_age_100_and_over: title: Diseases of the respiratory system_female_age_100_and_over - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_digestive_system_female_age_all_ages: title: Diseases of the digestive system_female_age_all ages - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_under_1: title: Diseases of the digestive system_female_age_under_1 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_1_4: title: Diseases of the digestive system_female_age_1-4 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_5_9: title: Diseases of the digestive system_female_age_5-9 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_10_14: title: Diseases of the digestive system_female_age_10-14 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_15_19: title: Diseases of the digestive system_female_age_15-19 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_20_24: title: Diseases of the digestive system_female_age_20-24 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_25_29: title: Diseases of the digestive system_female_age_25-29 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_30_34: title: Diseases of the digestive system_female_age_30-34 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_35_39: title: Diseases of the digestive system_female_age_35-39 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_40_44: title: Diseases of the digestive system_female_age_40-44 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_45_49: title: Diseases of the digestive system_female_age_45-49 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_50_54: title: Diseases of the digestive system_female_age_50-54 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_55_59: title: Diseases of the digestive system_female_age_55-59 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_60_64: title: Diseases of the digestive system_female_age_60-64 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_65_69: title: Diseases of the digestive system_female_age_65-69 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_70_74: title: Diseases of the digestive system_female_age_70-74 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_75_79: title: Diseases of the digestive system_female_age_75-79 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_80_84: title: Diseases of the digestive system_female_age_80-84 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_85_89: title: Diseases of the digestive system_female_age_85-89 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_90_94: title: Diseases of the digestive system_female_age_90-94 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_95_99: title: Diseases of the digestive system_female_age_95-99 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_female_age_100_and_over: title: Diseases of the digestive system_female_age_100_and_over - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_genitourinary_system_female_age_all_ages: title: Diseases of the genitourinary system_female_age_all ages - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_under_1: title: Diseases of the genitourinary system_female_age_under_1 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_1_4: title: Diseases of the genitourinary system_female_age_1-4 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_5_9: title: Diseases of the genitourinary system_female_age_5-9 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_10_14: title: Diseases of the genitourinary system_female_age_10-14 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_15_19: title: Diseases of the genitourinary system_female_age_15-19 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_20_24: title: Diseases of the genitourinary system_female_age_20-24 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_25_29: title: Diseases of the genitourinary system_female_age_25-29 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_30_34: title: Diseases of the genitourinary system_female_age_30-34 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_35_39: title: Diseases of the genitourinary system_female_age_35-39 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_40_44: title: Diseases of the genitourinary system_female_age_40-44 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_45_49: title: Diseases of the genitourinary system_female_age_45-49 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_50_54: title: Diseases of the genitourinary system_female_age_50-54 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_55_59: title: Diseases of the genitourinary system_female_age_55-59 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_60_64: title: Diseases of the genitourinary system_female_age_60-64 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_65_69: title: Diseases of the genitourinary system_female_age_65-69 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_70_74: title: Diseases of the genitourinary system_female_age_70-74 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_75_79: title: Diseases of the genitourinary system_female_age_75-79 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_80_84: title: Diseases of the genitourinary system_female_age_80-84 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_85_89: title: Diseases of the genitourinary system_female_age_85-89 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_90_94: title: Diseases of the genitourinary system_female_age_90-94 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_95_99: title: Diseases of the genitourinary system_female_age_95-99 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_female_age_100_and_over: title: Diseases of the genitourinary system_female_age_100_and_over - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_all_ages: title: Complications of pregnancy childbirth and the puerperium_female_age_all ages - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_under_1: title: Complications of pregnancy childbirth and the puerperium_female_age_under_1 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_1_4: title: Complications of pregnancy childbirth and the puerperium_female_age_1-4 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_5_9: title: Complications of pregnancy childbirth and the puerperium_female_age_5-9 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_10_14: title: Complications of pregnancy childbirth and the puerperium_female_age_10-14 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_15_19: title: Complications of pregnancy childbirth and the puerperium_female_age_15-19 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_20_24: title: Complications of pregnancy childbirth and the puerperium_female_age_20-24 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_25_29: title: Complications of pregnancy childbirth and the puerperium_female_age_25-29 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_30_34: title: Complications of pregnancy childbirth and the puerperium_female_age_30-34 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_35_39: title: Complications of pregnancy childbirth and the puerperium_female_age_35-39 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_40_44: title: Complications of pregnancy childbirth and the puerperium_female_age_40-44 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_45_49: title: Complications of pregnancy childbirth and the puerperium_female_age_45-49 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_50_54: title: Complications of pregnancy childbirth and the puerperium_female_age_50-54 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_55_59: title: Complications of pregnancy childbirth and the puerperium_female_age_55-59 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_60_64: title: Complications of pregnancy childbirth and the puerperium_female_age_60-64 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_65_69: title: Complications of pregnancy childbirth and the puerperium_female_age_65-69 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_70_74: title: Complications of pregnancy childbirth and the puerperium_female_age_70-74 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_75_79: title: Complications of pregnancy childbirth and the puerperium_female_age_75-79 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_80_84: title: Complications of pregnancy childbirth and the puerperium_female_age_80-84 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_85_89: title: Complications of pregnancy childbirth and the puerperium_female_age_85-89 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_90_94: title: Complications of pregnancy childbirth and the puerperium_female_age_90-94 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_95_99: title: Complications of pregnancy childbirth and the puerperium_female_age_95-99 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_female_age_100_and_over: title: Complications of pregnancy childbirth and the puerperium_female_age_100_and_over - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_all_ages: title: Diseases of the skin and subcutaneous tissue_female_age_all ages - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_under_1: title: Diseases of the skin and subcutaneous tissue_female_age_under_1 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_1_4: title: Diseases of the skin and subcutaneous tissue_female_age_1-4 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_5_9: title: Diseases of the skin and subcutaneous tissue_female_age_5-9 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_10_14: title: Diseases of the skin and subcutaneous tissue_female_age_10-14 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_15_19: title: Diseases of the skin and subcutaneous tissue_female_age_15-19 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_20_24: title: Diseases of the skin and subcutaneous tissue_female_age_20-24 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_25_29: title: Diseases of the skin and subcutaneous tissue_female_age_25-29 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_30_34: title: Diseases of the skin and subcutaneous tissue_female_age_30-34 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_35_39: title: Diseases of the skin and subcutaneous tissue_female_age_35-39 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_40_44: title: Diseases of the skin and subcutaneous tissue_female_age_40-44 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_45_49: title: Diseases of the skin and subcutaneous tissue_female_age_45-49 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_50_54: title: Diseases of the skin and subcutaneous tissue_female_age_50-54 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_55_59: title: Diseases of the skin and subcutaneous tissue_female_age_55-59 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_60_64: title: Diseases of the skin and subcutaneous tissue_female_age_60-64 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_65_69: title: Diseases of the skin and subcutaneous tissue_female_age_65-69 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_70_74: title: Diseases of the skin and subcutaneous tissue_female_age_70-74 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_75_79: title: Diseases of the skin and subcutaneous tissue_female_age_75-79 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_80_84: title: Diseases of the skin and subcutaneous tissue_female_age_80-84 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_85_89: title: Diseases of the skin and subcutaneous tissue_female_age_85-89 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_90_94: title: Diseases of the skin and subcutaneous tissue_female_age_90-94 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_95_99: title: Diseases of the skin and subcutaneous tissue_female_age_95-99 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_female_age_100_and_over: title: Diseases of the skin and subcutaneous tissue_female_age_100_and_over - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_all_ages: title: Diseases of the musculoskeletal system and connective tissue_female_age_all ages - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_under_1: title: Diseases of the musculoskeletal system and connective tissue_female_age_under_1 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_1_4: title: Diseases of the musculoskeletal system and connective tissue_female_age_1-4 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_5_9: title: Diseases of the musculoskeletal system and connective tissue_female_age_5-9 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_10_14: title: Diseases of the musculoskeletal system and connective tissue_female_age_10-14 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_15_19: title: Diseases of the musculoskeletal system and connective tissue_female_age_15-19 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_20_24: title: Diseases of the musculoskeletal system and connective tissue_female_age_20-24 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_25_29: title: Diseases of the musculoskeletal system and connective tissue_female_age_25-29 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_30_34: title: Diseases of the musculoskeletal system and connective tissue_female_age_30-34 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_35_39: title: Diseases of the musculoskeletal system and connective tissue_female_age_35-39 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_40_44: title: Diseases of the musculoskeletal system and connective tissue_female_age_40-44 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_45_49: title: Diseases of the musculoskeletal system and connective tissue_female_age_45-49 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_50_54: title: Diseases of the musculoskeletal system and connective tissue_female_age_50-54 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_55_59: title: Diseases of the musculoskeletal system and connective tissue_female_age_55-59 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_60_64: title: Diseases of the musculoskeletal system and connective tissue_female_age_60-64 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_65_69: title: Diseases of the musculoskeletal system and connective tissue_female_age_65-69 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_70_74: title: Diseases of the musculoskeletal system and connective tissue_female_age_70-74 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_75_79: title: Diseases of the musculoskeletal system and connective tissue_female_age_75-79 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_80_84: title: Diseases of the musculoskeletal system and connective tissue_female_age_80-84 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_85_89: title: Diseases of the musculoskeletal system and connective tissue_female_age_85-89 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_90_94: title: Diseases of the musculoskeletal system and connective tissue_female_age_90-94 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_95_99: title: Diseases of the musculoskeletal system and connective tissue_female_age_95-99 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_female_age_100_and_over: title: Diseases of the musculoskeletal system and connective tissue_female_age_100_and_over - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. congenital_anomalies_female_age_all_ages: title: Congenital anomalies_female_age_all ages - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_under_1: title: Congenital anomalies_female_age_under_1 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_1_4: title: Congenital anomalies_female_age_1-4 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_5_9: title: Congenital anomalies_female_age_5-9 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_10_14: title: Congenital anomalies_female_age_10-14 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_15_19: title: Congenital anomalies_female_age_15-19 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_20_24: title: Congenital anomalies_female_age_20-24 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_25_29: title: Congenital anomalies_female_age_25-29 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_30_34: title: Congenital anomalies_female_age_30-34 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_35_39: title: Congenital anomalies_female_age_35-39 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_40_44: title: Congenital anomalies_female_age_40-44 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_45_49: title: Congenital anomalies_female_age_45-49 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_50_54: title: Congenital anomalies_female_age_50-54 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_55_59: title: Congenital anomalies_female_age_55-59 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_60_64: title: Congenital anomalies_female_age_60-64 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_65_69: title: Congenital anomalies_female_age_65-69 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_70_74: title: Congenital anomalies_female_age_70-74 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_75_79: title: Congenital anomalies_female_age_75-79 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_80_84: title: Congenital anomalies_female_age_80-84 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_85_89: title: Congenital anomalies_female_age_85-89 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_90_94: title: Congenital anomalies_female_age_90-94 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_95_99: title: Congenital anomalies_female_age_95-99 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_female_age_100_and_over: title: Congenital anomalies_female_age_100_and_over - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. certain_conditions_originating_in_the_perinatal_period_female_age_all_ages: title: Certain conditions originating in the perinatal period_female_age_all ages - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_under_1: title: Certain conditions originating in the perinatal period_female_age_under_1 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_1_4: title: Certain conditions originating in the perinatal period_female_age_1-4 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_5_9: title: Certain conditions originating in the perinatal period_female_age_5-9 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_10_14: title: Certain conditions originating in the perinatal period_female_age_10-14 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_15_19: title: Certain conditions originating in the perinatal period_female_age_15-19 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_20_24: title: Certain conditions originating in the perinatal period_female_age_20-24 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_25_29: title: Certain conditions originating in the perinatal period_female_age_25-29 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_30_34: title: Certain conditions originating in the perinatal period_female_age_30-34 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_35_39: title: Certain conditions originating in the perinatal period_female_age_35-39 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_40_44: title: Certain conditions originating in the perinatal period_female_age_40-44 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_45_49: title: Certain conditions originating in the perinatal period_female_age_45-49 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_50_54: title: Certain conditions originating in the perinatal period_female_age_50-54 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_55_59: title: Certain conditions originating in the perinatal period_female_age_55-59 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_60_64: title: Certain conditions originating in the perinatal period_female_age_60-64 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_65_69: title: Certain conditions originating in the perinatal period_female_age_65-69 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_70_74: title: Certain conditions originating in the perinatal period_female_age_70-74 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_75_79: title: Certain conditions originating in the perinatal period_female_age_75-79 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_80_84: title: Certain conditions originating in the perinatal period_female_age_80-84 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_85_89: title: Certain conditions originating in the perinatal period_female_age_85-89 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_90_94: title: Certain conditions originating in the perinatal period_female_age_90-94 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_95_99: title: Certain conditions originating in the perinatal period_female_age_95-99 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_female_age_100_and_over: title: Certain conditions originating in the perinatal period_female_age_100_and_over - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. symptoms_signs_and_ill_defined_conditions_female_age_all_ages: title: Symptoms signs and ill-defined conditions_female_age_all ages - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_under_1: title: Symptoms signs and ill-defined conditions_female_age_under_1 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_1_4: title: Symptoms signs and ill-defined conditions_female_age_1-4 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_5_9: title: Symptoms signs and ill-defined conditions_female_age_5-9 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_10_14: title: Symptoms signs and ill-defined conditions_female_age_10-14 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_15_19: title: Symptoms signs and ill-defined conditions_female_age_15-19 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_20_24: title: Symptoms signs and ill-defined conditions_female_age_20-24 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_25_29: title: Symptoms signs and ill-defined conditions_female_age_25-29 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_30_34: title: Symptoms signs and ill-defined conditions_female_age_30-34 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_35_39: title: Symptoms signs and ill-defined conditions_female_age_35-39 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_40_44: title: Symptoms signs and ill-defined conditions_female_age_40-44 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_45_49: title: Symptoms signs and ill-defined conditions_female_age_45-49 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_50_54: title: Symptoms signs and ill-defined conditions_female_age_50-54 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_55_59: title: Symptoms signs and ill-defined conditions_female_age_55-59 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_60_64: title: Symptoms signs and ill-defined conditions_female_age_60-64 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_65_69: title: Symptoms signs and ill-defined conditions_female_age_65-69 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_70_74: title: Symptoms signs and ill-defined conditions_female_age_70-74 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_75_79: title: Symptoms signs and ill-defined conditions_female_age_75-79 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_80_84: title: Symptoms signs and ill-defined conditions_female_age_80-84 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_85_89: title: Symptoms signs and ill-defined conditions_female_age_85-89 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_90_94: title: Symptoms signs and ill-defined conditions_female_age_90-94 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_95_99: title: Symptoms signs and ill-defined conditions_female_age_95-99 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_female_age_100_and_over: title: Symptoms signs and ill-defined conditions_female_age_100_and_over - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. injury_and_poisoning_female_age_all_ages: title: Injury and poisoning_female_age_all ages - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_under_1: title: Injury and poisoning_female_age_under_1 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_1_4: title: Injury and poisoning_female_age_1-4 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_5_9: title: Injury and poisoning_female_age_5-9 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_10_14: title: Injury and poisoning_female_age_10-14 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_15_19: title: Injury and poisoning_female_age_15-19 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_20_24: title: Injury and poisoning_female_age_20-24 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_25_29: title: Injury and poisoning_female_age_25-29 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_30_34: title: Injury and poisoning_female_age_30-34 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_35_39: title: Injury and poisoning_female_age_35-39 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_40_44: title: Injury and poisoning_female_age_40-44 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_45_49: title: Injury and poisoning_female_age_45-49 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_50_54: title: Injury and poisoning_female_age_50-54 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_55_59: title: Injury and poisoning_female_age_55-59 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_60_64: title: Injury and poisoning_female_age_60-64 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_65_69: title: Injury and poisoning_female_age_65-69 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_70_74: title: Injury and poisoning_female_age_70-74 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_75_79: title: Injury and poisoning_female_age_75-79 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_80_84: title: Injury and poisoning_female_age_80-84 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_85_89: title: Injury and poisoning_female_age_85-89 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_90_94: title: Injury and poisoning_female_age_90-94 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_95_99: title: Injury and poisoning_female_age_95-99 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_female_age_100_and_over: title: Injury and poisoning_female_age_100_and_over - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. all_causes_male_age_all_ages: title: All causes_male_age_all ages - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_under_1: title: All causes_male_age_under_1 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_1_4: title: All causes_male_age_1-4 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_5_9: title: All causes_male_age_5-9 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_10_14: title: All causes_male_age_10-14 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_15_19: title: All causes_male_age_15-19 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_20_24: title: All causes_male_age_20-24 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_25_29: title: All causes_male_age_25-29 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_30_34: title: All causes_male_age_30-34 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_35_39: title: All causes_male_age_35-39 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_40_44: title: All causes_male_age_40-44 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_45_49: title: All causes_male_age_45-49 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_50_54: title: All causes_male_age_50-54 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_55_59: title: All causes_male_age_55-59 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_60_64: title: All causes_male_age_60-64 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_65_69: title: All causes_male_age_65-69 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_70_74: title: All causes_male_age_70-74 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_75_79: title: All causes_male_age_75-79 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_80_84: title: All causes_male_age_80-84 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_85_89: title: All causes_male_age_85-89 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_90_94: title: All causes_male_age_90-94 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_95_99: title: All causes_male_age_95-99 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_male_age_100_and_over: title: All causes_male_age_100_and_over - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. infectious_and_parasitic_diseases_male_age_all_ages: title: Infectious and parasitic diseases_male_age_all ages - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_under_1: title: Infectious and parasitic diseases_male_age_under_1 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_1_4: title: Infectious and parasitic diseases_male_age_1-4 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_5_9: title: Infectious and parasitic diseases_male_age_5-9 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_10_14: title: Infectious and parasitic diseases_male_age_10-14 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_15_19: title: Infectious and parasitic diseases_male_age_15-19 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_20_24: title: Infectious and parasitic diseases_male_age_20-24 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_25_29: title: Infectious and parasitic diseases_male_age_25-29 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_30_34: title: Infectious and parasitic diseases_male_age_30-34 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_35_39: title: Infectious and parasitic diseases_male_age_35-39 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_40_44: title: Infectious and parasitic diseases_male_age_40-44 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_45_49: title: Infectious and parasitic diseases_male_age_45-49 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_50_54: title: Infectious and parasitic diseases_male_age_50-54 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_55_59: title: Infectious and parasitic diseases_male_age_55-59 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_60_64: title: Infectious and parasitic diseases_male_age_60-64 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_65_69: title: Infectious and parasitic diseases_male_age_65-69 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_70_74: title: Infectious and parasitic diseases_male_age_70-74 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_75_79: title: Infectious and parasitic diseases_male_age_75-79 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_80_84: title: Infectious and parasitic diseases_male_age_80-84 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_85_89: title: Infectious and parasitic diseases_male_age_85-89 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_90_94: title: Infectious and parasitic diseases_male_age_90-94 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_95_99: title: Infectious and parasitic diseases_male_age_95-99 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_male_age_100_and_over: title: Infectious and parasitic diseases_male_age_100_and_over - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. neoplasms_male_age_all_ages: title: Neoplasms_male_age_all ages + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_under_1: title: Neoplasms_male_age_under_1 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_1_4: title: Neoplasms_male_age_1-4 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_5_9: title: Neoplasms_male_age_5-9 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_10_14: title: Neoplasms_male_age_10-14 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_15_19: title: Neoplasms_male_age_15-19 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_20_24: title: Neoplasms_male_age_20-24 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_25_29: title: Neoplasms_male_age_25-29 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_30_34: title: Neoplasms_male_age_30-34 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_35_39: title: Neoplasms_male_age_35-39 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_40_44: title: Neoplasms_male_age_40-44 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_45_49: title: Neoplasms_male_age_45-49 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_50_54: title: Neoplasms_male_age_50-54 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_55_59: title: Neoplasms_male_age_55-59 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_60_64: title: Neoplasms_male_age_60-64 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_65_69: title: Neoplasms_male_age_65-69 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_70_74: title: Neoplasms_male_age_70-74 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_75_79: title: Neoplasms_male_age_75-79 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_80_84: title: Neoplasms_male_age_80-84 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_85_89: title: Neoplasms_male_age_85-89 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_90_94: title: Neoplasms_male_age_90-94 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_95_99: title: Neoplasms_male_age_95-99 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_male_age_100_and_over: title: Neoplasms_male_age_100_and_over + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_all_ages: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_all ages - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_under_1: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_under_1 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_1_4: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_1-4 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_5_9: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_5-9 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_10_14: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_10-14 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_15_19: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_15-19 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_20_24: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_20-24 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_25_29: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_25-29 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_30_34: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_30-34 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_35_39: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_35-39 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_40_44: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_40-44 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_45_49: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_45-49 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_50_54: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_50-54 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_55_59: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_55-59 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_60_64: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_60-64 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_65_69: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_65-69 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_70_74: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_70-74 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_75_79: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_75-79 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_80_84: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_80-84 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_85_89: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_85-89 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_90_94: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_90-94 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_95_99: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_95-99 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_male_age_100_and_over: title: Endocrine nutritional and metabolic diseases and immunity disorders_male_age_100_and_over - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. diseases_of_the_blood_and_blood_forming_organs_male_age_all_ages: title: Diseases of the blood and blood-forming organs_male_age_all ages - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_under_1: title: Diseases of the blood and blood-forming organs_male_age_under_1 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_1_4: title: Diseases of the blood and blood-forming organs_male_age_1-4 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_5_9: title: Diseases of the blood and blood-forming organs_male_age_5-9 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_10_14: title: Diseases of the blood and blood-forming organs_male_age_10-14 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_15_19: title: Diseases of the blood and blood-forming organs_male_age_15-19 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_20_24: title: Diseases of the blood and blood-forming organs_male_age_20-24 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_25_29: title: Diseases of the blood and blood-forming organs_male_age_25-29 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_30_34: title: Diseases of the blood and blood-forming organs_male_age_30-34 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_35_39: title: Diseases of the blood and blood-forming organs_male_age_35-39 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_40_44: title: Diseases of the blood and blood-forming organs_male_age_40-44 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_45_49: title: Diseases of the blood and blood-forming organs_male_age_45-49 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_50_54: title: Diseases of the blood and blood-forming organs_male_age_50-54 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_55_59: title: Diseases of the blood and blood-forming organs_male_age_55-59 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_60_64: title: Diseases of the blood and blood-forming organs_male_age_60-64 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_65_69: title: Diseases of the blood and blood-forming organs_male_age_65-69 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_70_74: title: Diseases of the blood and blood-forming organs_male_age_70-74 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_75_79: title: Diseases of the blood and blood-forming organs_male_age_75-79 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_80_84: title: Diseases of the blood and blood-forming organs_male_age_80-84 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_85_89: title: Diseases of the blood and blood-forming organs_male_age_85-89 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_90_94: title: Diseases of the blood and blood-forming organs_male_age_90-94 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_95_99: title: Diseases of the blood and blood-forming organs_male_age_95-99 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_male_age_100_and_over: title: Diseases of the blood and blood-forming organs_male_age_100_and_over - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. mental_disorders_male_age_all_ages: title: Mental disorders_male_age_all ages - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_under_1: title: Mental disorders_male_age_under_1 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_1_4: title: Mental disorders_male_age_1-4 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_5_9: title: Mental disorders_male_age_5-9 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_10_14: title: Mental disorders_male_age_10-14 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_15_19: title: Mental disorders_male_age_15-19 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_20_24: title: Mental disorders_male_age_20-24 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_25_29: title: Mental disorders_male_age_25-29 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_30_34: title: Mental disorders_male_age_30-34 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_35_39: title: Mental disorders_male_age_35-39 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_40_44: title: Mental disorders_male_age_40-44 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_45_49: title: Mental disorders_male_age_45-49 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_50_54: title: Mental disorders_male_age_50-54 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_55_59: title: Mental disorders_male_age_55-59 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_60_64: title: Mental disorders_male_age_60-64 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_65_69: title: Mental disorders_male_age_65-69 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_70_74: title: Mental disorders_male_age_70-74 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_75_79: title: Mental disorders_male_age_75-79 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_80_84: title: Mental disorders_male_age_80-84 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_85_89: title: Mental disorders_male_age_85-89 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_90_94: title: Mental disorders_male_age_90-94 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_95_99: title: Mental disorders_male_age_95-99 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_male_age_100_and_over: title: Mental disorders_male_age_100_and_over - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. diseases_of_the_nervous_system_male_age_all_ages: title: Diseases of the nervous system_male_age_all ages - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_under_1: title: Diseases of the nervous system_male_age_under_1 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_1_4: title: Diseases of the nervous system_male_age_1-4 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_5_9: title: Diseases of the nervous system_male_age_5-9 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_10_14: title: Diseases of the nervous system_male_age_10-14 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_15_19: title: Diseases of the nervous system_male_age_15-19 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_20_24: title: Diseases of the nervous system_male_age_20-24 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_25_29: title: Diseases of the nervous system_male_age_25-29 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_30_34: title: Diseases of the nervous system_male_age_30-34 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_35_39: title: Diseases of the nervous system_male_age_35-39 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_40_44: title: Diseases of the nervous system_male_age_40-44 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_45_49: title: Diseases of the nervous system_male_age_45-49 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_50_54: title: Diseases of the nervous system_male_age_50-54 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_55_59: title: Diseases of the nervous system_male_age_55-59 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_60_64: title: Diseases of the nervous system_male_age_60-64 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_65_69: title: Diseases of the nervous system_male_age_65-69 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_70_74: title: Diseases of the nervous system_male_age_70-74 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_75_79: title: Diseases of the nervous system_male_age_75-79 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_80_84: title: Diseases of the nervous system_male_age_80-84 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_85_89: title: Diseases of the nervous system_male_age_85-89 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_90_94: title: Diseases of the nervous system_male_age_90-94 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_95_99: title: Diseases of the nervous system_male_age_95-99 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_male_age_100_and_over: title: Diseases of the nervous system_male_age_100_and_over - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_sense_organs_male_age_all_ages: title: Diseases of the sense organs_male_age_all ages - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_under_1: title: Diseases of the sense organs_male_age_under_1 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_1_4: title: Diseases of the sense organs_male_age_1-4 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_5_9: title: Diseases of the sense organs_male_age_5-9 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_10_14: title: Diseases of the sense organs_male_age_10-14 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_15_19: title: Diseases of the sense organs_male_age_15-19 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_20_24: title: Diseases of the sense organs_male_age_20-24 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_25_29: title: Diseases of the sense organs_male_age_25-29 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_30_34: title: Diseases of the sense organs_male_age_30-34 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_35_39: title: Diseases of the sense organs_male_age_35-39 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_40_44: title: Diseases of the sense organs_male_age_40-44 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_45_49: title: Diseases of the sense organs_male_age_45-49 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_50_54: title: Diseases of the sense organs_male_age_50-54 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_55_59: title: Diseases of the sense organs_male_age_55-59 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_60_64: title: Diseases of the sense organs_male_age_60-64 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_65_69: title: Diseases of the sense organs_male_age_65-69 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_70_74: title: Diseases of the sense organs_male_age_70-74 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_75_79: title: Diseases of the sense organs_male_age_75-79 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_80_84: title: Diseases of the sense organs_male_age_80-84 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_85_89: title: Diseases of the sense organs_male_age_85-89 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_90_94: title: Diseases of the sense organs_male_age_90-94 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_95_99: title: Diseases of the sense organs_male_age_95-99 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_male_age_100_and_over: title: Diseases of the sense organs_male_age_100_and_over - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_circulatory_system_male_age_all_ages: title: Diseases of the circulatory system_male_age_all ages - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_under_1: title: Diseases of the circulatory system_male_age_under_1 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_1_4: title: Diseases of the circulatory system_male_age_1-4 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_5_9: title: Diseases of the circulatory system_male_age_5-9 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_10_14: title: Diseases of the circulatory system_male_age_10-14 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_15_19: title: Diseases of the circulatory system_male_age_15-19 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_20_24: title: Diseases of the circulatory system_male_age_20-24 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_25_29: title: Diseases of the circulatory system_male_age_25-29 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_30_34: title: Diseases of the circulatory system_male_age_30-34 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_35_39: title: Diseases of the circulatory system_male_age_35-39 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_40_44: title: Diseases of the circulatory system_male_age_40-44 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_45_49: title: Diseases of the circulatory system_male_age_45-49 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_50_54: title: Diseases of the circulatory system_male_age_50-54 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_55_59: title: Diseases of the circulatory system_male_age_55-59 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_60_64: title: Diseases of the circulatory system_male_age_60-64 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_65_69: title: Diseases of the circulatory system_male_age_65-69 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_70_74: title: Diseases of the circulatory system_male_age_70-74 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_75_79: title: Diseases of the circulatory system_male_age_75-79 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_80_84: title: Diseases of the circulatory system_male_age_80-84 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_85_89: title: Diseases of the circulatory system_male_age_85-89 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_90_94: title: Diseases of the circulatory system_male_age_90-94 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_95_99: title: Diseases of the circulatory system_male_age_95-99 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_male_age_100_and_over: title: Diseases of the circulatory system_male_age_100_and_over - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_respiratory_system_male_age_all_ages: title: Diseases of the respiratory system_male_age_all ages - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_under_1: title: Diseases of the respiratory system_male_age_under_1 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_1_4: title: Diseases of the respiratory system_male_age_1-4 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_5_9: title: Diseases of the respiratory system_male_age_5-9 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_10_14: title: Diseases of the respiratory system_male_age_10-14 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_15_19: title: Diseases of the respiratory system_male_age_15-19 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_20_24: title: Diseases of the respiratory system_male_age_20-24 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_25_29: title: Diseases of the respiratory system_male_age_25-29 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_30_34: title: Diseases of the respiratory system_male_age_30-34 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_35_39: title: Diseases of the respiratory system_male_age_35-39 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_40_44: title: Diseases of the respiratory system_male_age_40-44 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_45_49: title: Diseases of the respiratory system_male_age_45-49 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_50_54: title: Diseases of the respiratory system_male_age_50-54 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_55_59: title: Diseases of the respiratory system_male_age_55-59 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_60_64: title: Diseases of the respiratory system_male_age_60-64 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_65_69: title: Diseases of the respiratory system_male_age_65-69 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_70_74: title: Diseases of the respiratory system_male_age_70-74 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_75_79: title: Diseases of the respiratory system_male_age_75-79 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_80_84: title: Diseases of the respiratory system_male_age_80-84 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_85_89: title: Diseases of the respiratory system_male_age_85-89 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_90_94: title: Diseases of the respiratory system_male_age_90-94 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_95_99: title: Diseases of the respiratory system_male_age_95-99 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_male_age_100_and_over: title: Diseases of the respiratory system_male_age_100_and_over - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_digestive_system_male_age_all_ages: title: Diseases of the digestive system_male_age_all ages - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_under_1: title: Diseases of the digestive system_male_age_under_1 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_1_4: title: Diseases of the digestive system_male_age_1-4 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_5_9: title: Diseases of the digestive system_male_age_5-9 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_10_14: title: Diseases of the digestive system_male_age_10-14 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_15_19: title: Diseases of the digestive system_male_age_15-19 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_20_24: title: Diseases of the digestive system_male_age_20-24 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_25_29: title: Diseases of the digestive system_male_age_25-29 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_30_34: title: Diseases of the digestive system_male_age_30-34 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_35_39: title: Diseases of the digestive system_male_age_35-39 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_40_44: title: Diseases of the digestive system_male_age_40-44 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_45_49: title: Diseases of the digestive system_male_age_45-49 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_50_54: title: Diseases of the digestive system_male_age_50-54 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_55_59: title: Diseases of the digestive system_male_age_55-59 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_60_64: title: Diseases of the digestive system_male_age_60-64 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_65_69: title: Diseases of the digestive system_male_age_65-69 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_70_74: title: Diseases of the digestive system_male_age_70-74 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_75_79: title: Diseases of the digestive system_male_age_75-79 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_80_84: title: Diseases of the digestive system_male_age_80-84 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_85_89: title: Diseases of the digestive system_male_age_85-89 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_90_94: title: Diseases of the digestive system_male_age_90-94 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_95_99: title: Diseases of the digestive system_male_age_95-99 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_male_age_100_and_over: title: Diseases of the digestive system_male_age_100_and_over - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_genitourinary_system_male_age_all_ages: title: Diseases of the genitourinary system_male_age_all ages - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_under_1: title: Diseases of the genitourinary system_male_age_under_1 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_1_4: title: Diseases of the genitourinary system_male_age_1-4 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_5_9: title: Diseases of the genitourinary system_male_age_5-9 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_10_14: title: Diseases of the genitourinary system_male_age_10-14 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_15_19: title: Diseases of the genitourinary system_male_age_15-19 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_20_24: title: Diseases of the genitourinary system_male_age_20-24 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_25_29: title: Diseases of the genitourinary system_male_age_25-29 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_30_34: title: Diseases of the genitourinary system_male_age_30-34 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_35_39: title: Diseases of the genitourinary system_male_age_35-39 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_40_44: title: Diseases of the genitourinary system_male_age_40-44 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_45_49: title: Diseases of the genitourinary system_male_age_45-49 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_50_54: title: Diseases of the genitourinary system_male_age_50-54 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_55_59: title: Diseases of the genitourinary system_male_age_55-59 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_60_64: title: Diseases of the genitourinary system_male_age_60-64 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_65_69: title: Diseases of the genitourinary system_male_age_65-69 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_70_74: title: Diseases of the genitourinary system_male_age_70-74 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_75_79: title: Diseases of the genitourinary system_male_age_75-79 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_80_84: title: Diseases of the genitourinary system_male_age_80-84 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_85_89: title: Diseases of the genitourinary system_male_age_85-89 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_90_94: title: Diseases of the genitourinary system_male_age_90-94 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_95_99: title: Diseases of the genitourinary system_male_age_95-99 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_male_age_100_and_over: title: Diseases of the genitourinary system_male_age_100_and_over - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_all_ages: title: Complications of pregnancy childbirth and the puerperium_male_age_all ages - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_under_1: title: Complications of pregnancy childbirth and the puerperium_male_age_under_1 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_1_4: title: Complications of pregnancy childbirth and the puerperium_male_age_1-4 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_5_9: title: Complications of pregnancy childbirth and the puerperium_male_age_5-9 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_10_14: title: Complications of pregnancy childbirth and the puerperium_male_age_10-14 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_15_19: title: Complications of pregnancy childbirth and the puerperium_male_age_15-19 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_20_24: title: Complications of pregnancy childbirth and the puerperium_male_age_20-24 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_25_29: title: Complications of pregnancy childbirth and the puerperium_male_age_25-29 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_30_34: title: Complications of pregnancy childbirth and the puerperium_male_age_30-34 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_35_39: title: Complications of pregnancy childbirth and the puerperium_male_age_35-39 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_40_44: title: Complications of pregnancy childbirth and the puerperium_male_age_40-44 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_45_49: title: Complications of pregnancy childbirth and the puerperium_male_age_45-49 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_50_54: title: Complications of pregnancy childbirth and the puerperium_male_age_50-54 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_55_59: title: Complications of pregnancy childbirth and the puerperium_male_age_55-59 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_60_64: title: Complications of pregnancy childbirth and the puerperium_male_age_60-64 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_65_69: title: Complications of pregnancy childbirth and the puerperium_male_age_65-69 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_70_74: title: Complications of pregnancy childbirth and the puerperium_male_age_70-74 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_75_79: title: Complications of pregnancy childbirth and the puerperium_male_age_75-79 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_80_84: title: Complications of pregnancy childbirth and the puerperium_male_age_80-84 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_85_89: title: Complications of pregnancy childbirth and the puerperium_male_age_85-89 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_90_94: title: Complications of pregnancy childbirth and the puerperium_male_age_90-94 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_95_99: title: Complications of pregnancy childbirth and the puerperium_male_age_95-99 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_male_age_100_and_over: title: Complications of pregnancy childbirth and the puerperium_male_age_100_and_over - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_all_ages: title: Diseases of the skin and subcutaneous tissue_male_age_all ages - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_under_1: title: Diseases of the skin and subcutaneous tissue_male_age_under_1 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_1_4: title: Diseases of the skin and subcutaneous tissue_male_age_1-4 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_5_9: title: Diseases of the skin and subcutaneous tissue_male_age_5-9 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_10_14: title: Diseases of the skin and subcutaneous tissue_male_age_10-14 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_15_19: title: Diseases of the skin and subcutaneous tissue_male_age_15-19 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_20_24: title: Diseases of the skin and subcutaneous tissue_male_age_20-24 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_25_29: title: Diseases of the skin and subcutaneous tissue_male_age_25-29 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_30_34: title: Diseases of the skin and subcutaneous tissue_male_age_30-34 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_35_39: title: Diseases of the skin and subcutaneous tissue_male_age_35-39 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_40_44: title: Diseases of the skin and subcutaneous tissue_male_age_40-44 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_45_49: title: Diseases of the skin and subcutaneous tissue_male_age_45-49 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_50_54: title: Diseases of the skin and subcutaneous tissue_male_age_50-54 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_55_59: title: Diseases of the skin and subcutaneous tissue_male_age_55-59 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_60_64: title: Diseases of the skin and subcutaneous tissue_male_age_60-64 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_65_69: title: Diseases of the skin and subcutaneous tissue_male_age_65-69 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_70_74: title: Diseases of the skin and subcutaneous tissue_male_age_70-74 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_75_79: title: Diseases of the skin and subcutaneous tissue_male_age_75-79 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_80_84: title: Diseases of the skin and subcutaneous tissue_male_age_80-84 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_85_89: title: Diseases of the skin and subcutaneous tissue_male_age_85-89 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_90_94: title: Diseases of the skin and subcutaneous tissue_male_age_90-94 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_95_99: title: Diseases of the skin and subcutaneous tissue_male_age_95-99 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_male_age_100_and_over: title: Diseases of the skin and subcutaneous tissue_male_age_100_and_over - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_all_ages: title: Diseases of the musculoskeletal system and connective tissue_male_age_all ages - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_under_1: title: Diseases of the musculoskeletal system and connective tissue_male_age_under_1 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_1_4: title: Diseases of the musculoskeletal system and connective tissue_male_age_1-4 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_5_9: title: Diseases of the musculoskeletal system and connective tissue_male_age_5-9 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_10_14: title: Diseases of the musculoskeletal system and connective tissue_male_age_10-14 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_15_19: title: Diseases of the musculoskeletal system and connective tissue_male_age_15-19 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_20_24: title: Diseases of the musculoskeletal system and connective tissue_male_age_20-24 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_25_29: title: Diseases of the musculoskeletal system and connective tissue_male_age_25-29 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_30_34: title: Diseases of the musculoskeletal system and connective tissue_male_age_30-34 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_35_39: title: Diseases of the musculoskeletal system and connective tissue_male_age_35-39 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_40_44: title: Diseases of the musculoskeletal system and connective tissue_male_age_40-44 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_45_49: title: Diseases of the musculoskeletal system and connective tissue_male_age_45-49 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_50_54: title: Diseases of the musculoskeletal system and connective tissue_male_age_50-54 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_55_59: title: Diseases of the musculoskeletal system and connective tissue_male_age_55-59 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_60_64: title: Diseases of the musculoskeletal system and connective tissue_male_age_60-64 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_65_69: title: Diseases of the musculoskeletal system and connective tissue_male_age_65-69 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_70_74: title: Diseases of the musculoskeletal system and connective tissue_male_age_70-74 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_75_79: title: Diseases of the musculoskeletal system and connective tissue_male_age_75-79 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_80_84: title: Diseases of the musculoskeletal system and connective tissue_male_age_80-84 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_85_89: title: Diseases of the musculoskeletal system and connective tissue_male_age_85-89 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_90_94: title: Diseases of the musculoskeletal system and connective tissue_male_age_90-94 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_95_99: title: Diseases of the musculoskeletal system and connective tissue_male_age_95-99 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_male_age_100_and_over: title: Diseases of the musculoskeletal system and connective tissue_male_age_100_and_over - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. congenital_anomalies_male_age_all_ages: title: Congenital anomalies_male_age_all ages - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_under_1: title: Congenital anomalies_male_age_under_1 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_1_4: title: Congenital anomalies_male_age_1-4 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_5_9: title: Congenital anomalies_male_age_5-9 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_10_14: title: Congenital anomalies_male_age_10-14 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_15_19: title: Congenital anomalies_male_age_15-19 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_20_24: title: Congenital anomalies_male_age_20-24 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_25_29: title: Congenital anomalies_male_age_25-29 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_30_34: title: Congenital anomalies_male_age_30-34 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_35_39: title: Congenital anomalies_male_age_35-39 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_40_44: title: Congenital anomalies_male_age_40-44 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_45_49: title: Congenital anomalies_male_age_45-49 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_50_54: title: Congenital anomalies_male_age_50-54 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_55_59: title: Congenital anomalies_male_age_55-59 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_60_64: title: Congenital anomalies_male_age_60-64 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_65_69: title: Congenital anomalies_male_age_65-69 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_70_74: title: Congenital anomalies_male_age_70-74 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_75_79: title: Congenital anomalies_male_age_75-79 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_80_84: title: Congenital anomalies_male_age_80-84 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_85_89: title: Congenital anomalies_male_age_85-89 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_90_94: title: Congenital anomalies_male_age_90-94 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_95_99: title: Congenital anomalies_male_age_95-99 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_male_age_100_and_over: title: Congenital anomalies_male_age_100_and_over - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. certain_conditions_originating_in_the_perinatal_period_male_age_all_ages: title: Certain conditions originating in the perinatal period_male_age_all ages - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_under_1: title: Certain conditions originating in the perinatal period_male_age_under_1 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_1_4: title: Certain conditions originating in the perinatal period_male_age_1-4 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_5_9: title: Certain conditions originating in the perinatal period_male_age_5-9 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_10_14: title: Certain conditions originating in the perinatal period_male_age_10-14 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_15_19: title: Certain conditions originating in the perinatal period_male_age_15-19 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_20_24: title: Certain conditions originating in the perinatal period_male_age_20-24 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_25_29: title: Certain conditions originating in the perinatal period_male_age_25-29 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_30_34: title: Certain conditions originating in the perinatal period_male_age_30-34 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_35_39: title: Certain conditions originating in the perinatal period_male_age_35-39 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_40_44: title: Certain conditions originating in the perinatal period_male_age_40-44 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_45_49: title: Certain conditions originating in the perinatal period_male_age_45-49 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_50_54: title: Certain conditions originating in the perinatal period_male_age_50-54 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_55_59: title: Certain conditions originating in the perinatal period_male_age_55-59 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_60_64: title: Certain conditions originating in the perinatal period_male_age_60-64 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_65_69: title: Certain conditions originating in the perinatal period_male_age_65-69 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_70_74: title: Certain conditions originating in the perinatal period_male_age_70-74 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_75_79: title: Certain conditions originating in the perinatal period_male_age_75-79 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_80_84: title: Certain conditions originating in the perinatal period_male_age_80-84 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_85_89: title: Certain conditions originating in the perinatal period_male_age_85-89 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_90_94: title: Certain conditions originating in the perinatal period_male_age_90-94 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_95_99: title: Certain conditions originating in the perinatal period_male_age_95-99 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_male_age_100_and_over: title: Certain conditions originating in the perinatal period_male_age_100_and_over - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. symptoms_signs_and_ill_defined_conditions_male_age_all_ages: title: Symptoms signs and ill-defined conditions_male_age_all ages - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_under_1: title: Symptoms signs and ill-defined conditions_male_age_under_1 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_1_4: title: Symptoms signs and ill-defined conditions_male_age_1-4 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_5_9: title: Symptoms signs and ill-defined conditions_male_age_5-9 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_10_14: title: Symptoms signs and ill-defined conditions_male_age_10-14 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_15_19: title: Symptoms signs and ill-defined conditions_male_age_15-19 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_20_24: title: Symptoms signs and ill-defined conditions_male_age_20-24 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_25_29: title: Symptoms signs and ill-defined conditions_male_age_25-29 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_30_34: title: Symptoms signs and ill-defined conditions_male_age_30-34 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_35_39: title: Symptoms signs and ill-defined conditions_male_age_35-39 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_40_44: title: Symptoms signs and ill-defined conditions_male_age_40-44 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_45_49: title: Symptoms signs and ill-defined conditions_male_age_45-49 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_50_54: title: Symptoms signs and ill-defined conditions_male_age_50-54 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_55_59: title: Symptoms signs and ill-defined conditions_male_age_55-59 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_60_64: title: Symptoms signs and ill-defined conditions_male_age_60-64 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_65_69: title: Symptoms signs and ill-defined conditions_male_age_65-69 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_70_74: title: Symptoms signs and ill-defined conditions_male_age_70-74 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_75_79: title: Symptoms signs and ill-defined conditions_male_age_75-79 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_80_84: title: Symptoms signs and ill-defined conditions_male_age_80-84 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_85_89: title: Symptoms signs and ill-defined conditions_male_age_85-89 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_90_94: title: Symptoms signs and ill-defined conditions_male_age_90-94 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_95_99: title: Symptoms signs and ill-defined conditions_male_age_95-99 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_male_age_100_and_over: title: Symptoms signs and ill-defined conditions_male_age_100_and_over - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. injury_and_poisoning_male_age_all_ages: title: Injury and poisoning_male_age_all ages - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_under_1: title: Injury and poisoning_male_age_under_1 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_1_4: title: Injury and poisoning_male_age_1-4 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_5_9: title: Injury and poisoning_male_age_5-9 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_10_14: title: Injury and poisoning_male_age_10-14 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_15_19: title: Injury and poisoning_male_age_15-19 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_20_24: title: Injury and poisoning_male_age_20-24 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_25_29: title: Injury and poisoning_male_age_25-29 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_30_34: title: Injury and poisoning_male_age_30-34 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_35_39: title: Injury and poisoning_male_age_35-39 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_40_44: title: Injury and poisoning_male_age_40-44 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_45_49: title: Injury and poisoning_male_age_45-49 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_50_54: title: Injury and poisoning_male_age_50-54 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_55_59: title: Injury and poisoning_male_age_55-59 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_60_64: title: Injury and poisoning_male_age_60-64 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_65_69: title: Injury and poisoning_male_age_65-69 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_70_74: title: Injury and poisoning_male_age_70-74 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_75_79: title: Injury and poisoning_male_age_75-79 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_80_84: title: Injury and poisoning_male_age_80-84 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_85_89: title: Injury and poisoning_male_age_85-89 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_90_94: title: Injury and poisoning_male_age_90-94 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_95_99: title: Injury and poisoning_male_age_95-99 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_male_age_100_and_over: title: Injury and poisoning_male_age_100_and_over - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. all_causes_both_sexes_age_all_ages: title: All causes_both sexes_age_all ages - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_under_1: title: All causes_both sexes_age_under_1 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_1_4: title: All causes_both sexes_age_1-4 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_5_9: title: All causes_both sexes_age_5-9 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_10_14: title: All causes_both sexes_age_10-14 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_15_19: title: All causes_both sexes_age_15-19 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_20_24: title: All causes_both sexes_age_20-24 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_25_29: title: All causes_both sexes_age_25-29 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_30_34: title: All causes_both sexes_age_30-34 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_35_39: title: All causes_both sexes_age_35-39 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_40_44: title: All causes_both sexes_age_40-44 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_45_49: title: All causes_both sexes_age_45-49 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_50_54: title: All causes_both sexes_age_50-54 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_55_59: title: All causes_both sexes_age_55-59 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_60_64: title: All causes_both sexes_age_60-64 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_65_69: title: All causes_both sexes_age_65-69 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_70_74: title: All causes_both sexes_age_70-74 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_75_79: title: All causes_both sexes_age_75-79 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_80_84: title: All causes_both sexes_age_80-84 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_85_89: title: All causes_both sexes_age_85-89 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_90_94: title: All causes_both sexes_age_90-94 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_95_99: title: All causes_both sexes_age_95-99 - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. all_causes_both_sexes_age_100_and_over: title: All causes_both sexes_age_100_and_over - description: The mortality rate from all causes per 100 000 people in the whole population. unit: per 100 000 + description: The mortality rate from all causes per 100 000 people in the whole population. infectious_and_parasitic_diseases_both_sexes_age_all_ages: title: Infectious and parasitic diseases_both sexes_age_all ages - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_under_1: title: Infectious and parasitic diseases_both sexes_age_under_1 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_1_4: title: Infectious and parasitic diseases_both sexes_age_1-4 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_5_9: title: Infectious and parasitic diseases_both sexes_age_5-9 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_10_14: title: Infectious and parasitic diseases_both sexes_age_10-14 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_15_19: title: Infectious and parasitic diseases_both sexes_age_15-19 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_20_24: title: Infectious and parasitic diseases_both sexes_age_20-24 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_25_29: title: Infectious and parasitic diseases_both sexes_age_25-29 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_30_34: title: Infectious and parasitic diseases_both sexes_age_30-34 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_35_39: title: Infectious and parasitic diseases_both sexes_age_35-39 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_40_44: title: Infectious and parasitic diseases_both sexes_age_40-44 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_45_49: title: Infectious and parasitic diseases_both sexes_age_45-49 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_50_54: title: Infectious and parasitic diseases_both sexes_age_50-54 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_55_59: title: Infectious and parasitic diseases_both sexes_age_55-59 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_60_64: title: Infectious and parasitic diseases_both sexes_age_60-64 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_65_69: title: Infectious and parasitic diseases_both sexes_age_65-69 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_70_74: title: Infectious and parasitic diseases_both sexes_age_70-74 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_75_79: title: Infectious and parasitic diseases_both sexes_age_75-79 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_80_84: title: Infectious and parasitic diseases_both sexes_age_80-84 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_85_89: title: Infectious and parasitic diseases_both sexes_age_85-89 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_90_94: title: Infectious and parasitic diseases_both sexes_age_90-94 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_95_99: title: Infectious and parasitic diseases_both sexes_age_95-99 - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. infectious_and_parasitic_diseases_both_sexes_age_100_and_over: title: Infectious and parasitic diseases_both sexes_age_100_and_over - description: The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. - Infectious and parasitic diseases have the ICD death codes 001*-139*. unit: per 100 000 + description: |- + The mortality rate from infectious and parasitic diseases per 100 000 people in the whole population. Infectious and parasitic diseases have the ICD death codes 001*-139*. neoplasms_both_sexes_age_all_ages: title: Neoplasms_both sexes_age_all ages + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_under_1: title: Neoplasms_both sexes_age_under_1 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_1_4: title: Neoplasms_both sexes_age_1-4 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_5_9: title: Neoplasms_both sexes_age_5-9 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_10_14: title: Neoplasms_both sexes_age_10-14 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_15_19: title: Neoplasms_both sexes_age_15-19 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_20_24: title: Neoplasms_both sexes_age_20-24 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_25_29: title: Neoplasms_both sexes_age_25-29 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_30_34: title: Neoplasms_both sexes_age_30-34 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_35_39: title: Neoplasms_both sexes_age_35-39 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_40_44: title: Neoplasms_both sexes_age_40-44 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_45_49: title: Neoplasms_both sexes_age_45-49 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_50_54: title: Neoplasms_both sexes_age_50-54 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_55_59: title: Neoplasms_both sexes_age_55-59 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_60_64: title: Neoplasms_both sexes_age_60-64 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_65_69: title: Neoplasms_both sexes_age_65-69 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_70_74: title: Neoplasms_both sexes_age_70-74 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_75_79: title: Neoplasms_both sexes_age_75-79 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_80_84: title: Neoplasms_both sexes_age_80-84 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_85_89: title: Neoplasms_both sexes_age_85-89 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_90_94: title: Neoplasms_both sexes_age_90-94 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_95_99: title: Neoplasms_both sexes_age_95-99 + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 neoplasms_both_sexes_age_100_and_over: title: Neoplasms_both sexes_age_100_and_over + unit: per 100 000 description: The mortality rate from cancers per 100 000 people in the population. Cancers have the ICD death codes 140*-239*. - unit: per 100 000 endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_all_ages: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_all ages - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_under_1: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_under_1 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_1_4: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_1-4 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_5_9: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_5-9 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_10_14: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_10-14 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_15_19: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_15-19 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_20_24: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_20-24 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_25_29: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_25-29 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_30_34: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_30-34 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_35_39: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_35-39 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_40_44: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_40-44 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_45_49: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_45-49 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_50_54: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_50-54 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_55_59: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_55-59 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_60_64: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_60-64 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_65_69: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_65-69 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_70_74: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_70-74 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_75_79: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_75-79 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_80_84: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_80-84 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_85_89: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_85-89 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_90_94: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_90-94 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_95_99: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_95-99 - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. endocrine_nutritional_and_metabolic_diseases_and_immunity_disorders_both_sexes_age_100_and_over: title: Endocrine nutritional and metabolic diseases and immunity disorders_both sexes_age_100_and_over - description: The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 - 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD - death codes 240*-279*. unit: per 100 000 + description: |- + The mortality rate from endocrine, nutritional and metabolic diseases, and immunity disorders per 100 000 people in the population. Endocrine, nutritional and metabolic diseases, and immunity disorders have the ICD death codes 240*-279*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_all_ages: title: Diseases of the blood and blood-forming organs_both sexes_age_all ages - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_under_1: title: Diseases of the blood and blood-forming organs_both sexes_age_under_1 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_1_4: title: Diseases of the blood and blood-forming organs_both sexes_age_1-4 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_5_9: title: Diseases of the blood and blood-forming organs_both sexes_age_5-9 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_10_14: title: Diseases of the blood and blood-forming organs_both sexes_age_10-14 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_15_19: title: Diseases of the blood and blood-forming organs_both sexes_age_15-19 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_20_24: title: Diseases of the blood and blood-forming organs_both sexes_age_20-24 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_25_29: title: Diseases of the blood and blood-forming organs_both sexes_age_25-29 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_30_34: title: Diseases of the blood and blood-forming organs_both sexes_age_30-34 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_35_39: title: Diseases of the blood and blood-forming organs_both sexes_age_35-39 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_40_44: title: Diseases of the blood and blood-forming organs_both sexes_age_40-44 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_45_49: title: Diseases of the blood and blood-forming organs_both sexes_age_45-49 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_50_54: title: Diseases of the blood and blood-forming organs_both sexes_age_50-54 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_55_59: title: Diseases of the blood and blood-forming organs_both sexes_age_55-59 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_60_64: title: Diseases of the blood and blood-forming organs_both sexes_age_60-64 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_65_69: title: Diseases of the blood and blood-forming organs_both sexes_age_65-69 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_70_74: title: Diseases of the blood and blood-forming organs_both sexes_age_70-74 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_75_79: title: Diseases of the blood and blood-forming organs_both sexes_age_75-79 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_80_84: title: Diseases of the blood and blood-forming organs_both sexes_age_80-84 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_85_89: title: Diseases of the blood and blood-forming organs_both sexes_age_85-89 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_90_94: title: Diseases of the blood and blood-forming organs_both sexes_age_90-94 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_95_99: title: Diseases of the blood and blood-forming organs_both sexes_age_95-99 - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. diseases_of_the_blood_and_blood_forming_organs_both_sexes_age_100_and_over: title: Diseases of the blood and blood-forming organs_both sexes_age_100_and_over - description: The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. - Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. unit: per 100 000 + description: |- + The mortality rate from diseases of the blood and blood-forming organs per 100 000 people in the population. Diseases of the blood and blood-forming organs have the ICD death codes 280*-289*. mental_disorders_both_sexes_age_all_ages: title: Mental disorders_both sexes_age_all ages - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_under_1: title: Mental disorders_both sexes_age_under_1 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_1_4: title: Mental disorders_both sexes_age_1-4 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_5_9: title: Mental disorders_both sexes_age_5-9 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_10_14: title: Mental disorders_both sexes_age_10-14 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_15_19: title: Mental disorders_both sexes_age_15-19 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_20_24: title: Mental disorders_both sexes_age_20-24 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_25_29: title: Mental disorders_both sexes_age_25-29 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_30_34: title: Mental disorders_both sexes_age_30-34 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_35_39: title: Mental disorders_both sexes_age_35-39 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_40_44: title: Mental disorders_both sexes_age_40-44 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_45_49: title: Mental disorders_both sexes_age_45-49 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_50_54: title: Mental disorders_both sexes_age_50-54 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_55_59: title: Mental disorders_both sexes_age_55-59 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_60_64: title: Mental disorders_both sexes_age_60-64 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_65_69: title: Mental disorders_both sexes_age_65-69 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_70_74: title: Mental disorders_both sexes_age_70-74 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_75_79: title: Mental disorders_both sexes_age_75-79 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_80_84: title: Mental disorders_both sexes_age_80-84 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_85_89: title: Mental disorders_both sexes_age_85-89 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_90_94: title: Mental disorders_both sexes_age_90-94 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_95_99: title: Mental disorders_both sexes_age_95-99 - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. mental_disorders_both_sexes_age_100_and_over: title: Mental disorders_both sexes_age_100_and_over - description: The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have - the ICD death codes 290*-319*. unit: per 100 000 + description: |- + The mortality rate from mental illnesses per 100 000 people in the population. Mental illnesses have the ICD death codes 290*-319*. diseases_of_the_nervous_system_both_sexes_age_all_ages: title: Diseases of the nervous system_both sexes_age_all ages - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_under_1: title: Diseases of the nervous system_both sexes_age_under_1 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_1_4: title: Diseases of the nervous system_both sexes_age_1-4 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_5_9: title: Diseases of the nervous system_both sexes_age_5-9 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_10_14: title: Diseases of the nervous system_both sexes_age_10-14 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_15_19: title: Diseases of the nervous system_both sexes_age_15-19 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_20_24: title: Diseases of the nervous system_both sexes_age_20-24 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_25_29: title: Diseases of the nervous system_both sexes_age_25-29 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_30_34: title: Diseases of the nervous system_both sexes_age_30-34 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_35_39: title: Diseases of the nervous system_both sexes_age_35-39 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_40_44: title: Diseases of the nervous system_both sexes_age_40-44 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_45_49: title: Diseases of the nervous system_both sexes_age_45-49 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_50_54: title: Diseases of the nervous system_both sexes_age_50-54 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_55_59: title: Diseases of the nervous system_both sexes_age_55-59 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_60_64: title: Diseases of the nervous system_both sexes_age_60-64 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_65_69: title: Diseases of the nervous system_both sexes_age_65-69 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_70_74: title: Diseases of the nervous system_both sexes_age_70-74 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_75_79: title: Diseases of the nervous system_both sexes_age_75-79 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_80_84: title: Diseases of the nervous system_both sexes_age_80-84 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_85_89: title: Diseases of the nervous system_both sexes_age_85-89 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_90_94: title: Diseases of the nervous system_both sexes_age_90-94 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_95_99: title: Diseases of the nervous system_both sexes_age_95-99 - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_nervous_system_both_sexes_age_100_and_over: title: Diseases of the nervous system_both sexes_age_100_and_over - description: The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases - of the nervous system have the ICD death code 320*-359*. unit: per 100 000 + description: |- + The mortality rate from diseases of the nervous system per 100 000 people in the population. Diseases of the nervous system have the ICD death code 320*-359*. diseases_of_the_sense_organs_both_sexes_age_all_ages: title: Diseases of the sense organs_both sexes_age_all ages - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_under_1: title: Diseases of the sense organs_both sexes_age_under_1 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_1_4: title: Diseases of the sense organs_both sexes_age_1-4 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_5_9: title: Diseases of the sense organs_both sexes_age_5-9 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_10_14: title: Diseases of the sense organs_both sexes_age_10-14 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_15_19: title: Diseases of the sense organs_both sexes_age_15-19 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_20_24: title: Diseases of the sense organs_both sexes_age_20-24 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_25_29: title: Diseases of the sense organs_both sexes_age_25-29 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_30_34: title: Diseases of the sense organs_both sexes_age_30-34 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_35_39: title: Diseases of the sense organs_both sexes_age_35-39 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_40_44: title: Diseases of the sense organs_both sexes_age_40-44 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_45_49: title: Diseases of the sense organs_both sexes_age_45-49 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_50_54: title: Diseases of the sense organs_both sexes_age_50-54 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_55_59: title: Diseases of the sense organs_both sexes_age_55-59 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_60_64: title: Diseases of the sense organs_both sexes_age_60-64 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_65_69: title: Diseases of the sense organs_both sexes_age_65-69 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_70_74: title: Diseases of the sense organs_both sexes_age_70-74 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_75_79: title: Diseases of the sense organs_both sexes_age_75-79 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_80_84: title: Diseases of the sense organs_both sexes_age_80-84 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_85_89: title: Diseases of the sense organs_both sexes_age_85-89 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_90_94: title: Diseases of the sense organs_both sexes_age_90-94 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_95_99: title: Diseases of the sense organs_both sexes_age_95-99 - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_sense_organs_both_sexes_age_100_and_over: title: Diseases of the sense organs_both sexes_age_100_and_over - description: The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of - the nervous system have the ICD death code 360*-389*. unit: per 100 000 + description: |- + The mortality rate from diseases of the sense organs per 100 000 people in the population. Diseases of the nervous system have the ICD death code 360*-389*. diseases_of_the_circulatory_system_both_sexes_age_all_ages: title: Diseases of the circulatory system_both sexes_age_all ages - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_under_1: title: Diseases of the circulatory system_both sexes_age_under_1 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_1_4: title: Diseases of the circulatory system_both sexes_age_1-4 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_5_9: title: Diseases of the circulatory system_both sexes_age_5-9 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_10_14: title: Diseases of the circulatory system_both sexes_age_10-14 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_15_19: title: Diseases of the circulatory system_both sexes_age_15-19 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_20_24: title: Diseases of the circulatory system_both sexes_age_20-24 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_25_29: title: Diseases of the circulatory system_both sexes_age_25-29 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_30_34: title: Diseases of the circulatory system_both sexes_age_30-34 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_35_39: title: Diseases of the circulatory system_both sexes_age_35-39 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_40_44: title: Diseases of the circulatory system_both sexes_age_40-44 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_45_49: title: Diseases of the circulatory system_both sexes_age_45-49 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_50_54: title: Diseases of the circulatory system_both sexes_age_50-54 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_55_59: title: Diseases of the circulatory system_both sexes_age_55-59 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_60_64: title: Diseases of the circulatory system_both sexes_age_60-64 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_65_69: title: Diseases of the circulatory system_both sexes_age_65-69 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_70_74: title: Diseases of the circulatory system_both sexes_age_70-74 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_75_79: title: Diseases of the circulatory system_both sexes_age_75-79 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_80_84: title: Diseases of the circulatory system_both sexes_age_80-84 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_85_89: title: Diseases of the circulatory system_both sexes_age_85-89 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_90_94: title: Diseases of the circulatory system_both sexes_age_90-94 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_95_99: title: Diseases of the circulatory system_both sexes_age_95-99 - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_circulatory_system_both_sexes_age_100_and_over: title: Diseases of the circulatory system_both sexes_age_100_and_over - description: The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases - of the circulatory system have the ICD death codes 390*-459* unit: per 100 000 + description: |- + The mortality rate from diseases of the circulatory system per 100 000 people in the population. Diseases of the circulatory system have the ICD death codes 390*-459* diseases_of_the_respiratory_system_both_sexes_age_all_ages: title: Diseases of the respiratory system_both sexes_age_all ages - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_under_1: title: Diseases of the respiratory system_both sexes_age_under_1 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_1_4: title: Diseases of the respiratory system_both sexes_age_1-4 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_5_9: title: Diseases of the respiratory system_both sexes_age_5-9 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_10_14: title: Diseases of the respiratory system_both sexes_age_10-14 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_15_19: title: Diseases of the respiratory system_both sexes_age_15-19 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_20_24: title: Diseases of the respiratory system_both sexes_age_20-24 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_25_29: title: Diseases of the respiratory system_both sexes_age_25-29 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_30_34: title: Diseases of the respiratory system_both sexes_age_30-34 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_35_39: title: Diseases of the respiratory system_both sexes_age_35-39 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_40_44: title: Diseases of the respiratory system_both sexes_age_40-44 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_45_49: title: Diseases of the respiratory system_both sexes_age_45-49 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_50_54: title: Diseases of the respiratory system_both sexes_age_50-54 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_55_59: title: Diseases of the respiratory system_both sexes_age_55-59 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_60_64: title: Diseases of the respiratory system_both sexes_age_60-64 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_65_69: title: Diseases of the respiratory system_both sexes_age_65-69 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_70_74: title: Diseases of the respiratory system_both sexes_age_70-74 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_75_79: title: Diseases of the respiratory system_both sexes_age_75-79 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_80_84: title: Diseases of the respiratory system_both sexes_age_80-84 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_85_89: title: Diseases of the respiratory system_both sexes_age_85-89 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_90_94: title: Diseases of the respiratory system_both sexes_age_90-94 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_95_99: title: Diseases of the respiratory system_both sexes_age_95-99 - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_respiratory_system_both_sexes_age_100_and_over: title: Diseases of the respiratory system_both sexes_age_100_and_over - description: The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases - of the respiratory system have the ICD death codes 460*-519*. unit: per 100 000 + description: |- + The mortality rate from diseases of the respiratory system per 100 000 people in the population. Diseases of the respiratory system have the ICD death codes 460*-519*. diseases_of_the_digestive_system_both_sexes_age_all_ages: title: Diseases of the digestive system_both sexes_age_all ages - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_under_1: title: Diseases of the digestive system_both sexes_age_under_1 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_1_4: title: Diseases of the digestive system_both sexes_age_1-4 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_5_9: title: Diseases of the digestive system_both sexes_age_5-9 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_10_14: title: Diseases of the digestive system_both sexes_age_10-14 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_15_19: title: Diseases of the digestive system_both sexes_age_15-19 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_20_24: title: Diseases of the digestive system_both sexes_age_20-24 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_25_29: title: Diseases of the digestive system_both sexes_age_25-29 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_30_34: title: Diseases of the digestive system_both sexes_age_30-34 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_35_39: title: Diseases of the digestive system_both sexes_age_35-39 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_40_44: title: Diseases of the digestive system_both sexes_age_40-44 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_45_49: title: Diseases of the digestive system_both sexes_age_45-49 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_50_54: title: Diseases of the digestive system_both sexes_age_50-54 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_55_59: title: Diseases of the digestive system_both sexes_age_55-59 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_60_64: title: Diseases of the digestive system_both sexes_age_60-64 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_65_69: title: Diseases of the digestive system_both sexes_age_65-69 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_70_74: title: Diseases of the digestive system_both sexes_age_70-74 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_75_79: title: Diseases of the digestive system_both sexes_age_75-79 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_80_84: title: Diseases of the digestive system_both sexes_age_80-84 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_85_89: title: Diseases of the digestive system_both sexes_age_85-89 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_90_94: title: Diseases of the digestive system_both sexes_age_90-94 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_95_99: title: Diseases of the digestive system_both sexes_age_95-99 - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_digestive_system_both_sexes_age_100_and_over: title: Diseases of the digestive system_both sexes_age_100_and_over - description: The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases - of the digestive system have the ICD death codes 520*-579*. unit: per 100 000 + description: |- + The mortality rate from diseases of the digestive system per 100 000 people in the population. Diseases of the digestive system have the ICD death codes 520*-579*. diseases_of_the_genitourinary_system_both_sexes_age_all_ages: title: Diseases of the genitourinary system_both sexes_age_all ages - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_under_1: title: Diseases of the genitourinary system_both sexes_age_under_1 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_1_4: title: Diseases of the genitourinary system_both sexes_age_1-4 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_5_9: title: Diseases of the genitourinary system_both sexes_age_5-9 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_10_14: title: Diseases of the genitourinary system_both sexes_age_10-14 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_15_19: title: Diseases of the genitourinary system_both sexes_age_15-19 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_20_24: title: Diseases of the genitourinary system_both sexes_age_20-24 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_25_29: title: Diseases of the genitourinary system_both sexes_age_25-29 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_30_34: title: Diseases of the genitourinary system_both sexes_age_30-34 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_35_39: title: Diseases of the genitourinary system_both sexes_age_35-39 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_40_44: title: Diseases of the genitourinary system_both sexes_age_40-44 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_45_49: title: Diseases of the genitourinary system_both sexes_age_45-49 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_50_54: title: Diseases of the genitourinary system_both sexes_age_50-54 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_55_59: title: Diseases of the genitourinary system_both sexes_age_55-59 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_60_64: title: Diseases of the genitourinary system_both sexes_age_60-64 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_65_69: title: Diseases of the genitourinary system_both sexes_age_65-69 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_70_74: title: Diseases of the genitourinary system_both sexes_age_70-74 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_75_79: title: Diseases of the genitourinary system_both sexes_age_75-79 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_80_84: title: Diseases of the genitourinary system_both sexes_age_80-84 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_85_89: title: Diseases of the genitourinary system_both sexes_age_85-89 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_90_94: title: Diseases of the genitourinary system_both sexes_age_90-94 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_95_99: title: Diseases of the genitourinary system_both sexes_age_95-99 - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. diseases_of_the_genitourinary_system_both_sexes_age_100_and_over: title: Diseases of the genitourinary system_both sexes_age_100_and_over - description: The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases - of the genitourinary system have the ICD death codes 580*-629*. unit: per 100 000 + description: |- + The mortality rate from diseases of the genitourinary system per 100 000 people in the population. Diseases of the genitourinary system have the ICD death codes 580*-629*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_all_ages: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_all ages - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_under_1: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_under_1 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_1_4: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_1-4 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_5_9: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_5-9 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_10_14: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_10-14 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_15_19: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_15-19 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_20_24: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_20-24 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_25_29: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_25-29 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_30_34: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_30-34 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_35_39: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_35-39 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_40_44: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_40-44 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_45_49: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_45-49 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_50_54: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_50-54 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_55_59: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_55-59 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_60_64: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_60-64 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_65_69: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_65-69 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_70_74: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_70-74 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_75_79: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_75-79 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_80_84: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_80-84 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_85_89: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_85-89 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_90_94: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_90-94 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_95_99: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_95-99 - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. complications_of_pregnancy_childbirth_and_the_puerperium_both_sexes_age_100_and_over: title: Complications of pregnancy childbirth and the puerperium_both sexes_age_100_and_over - description: The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people - in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. unit: per 100 000 + description: |- + The mortality rate from complications of pregnancy, childbirth and the puerperium per 100 000 people in the population. Complications of pregnancy, childbirth and the puerperium have the ICD death codes 630*-679*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_all_ages: title: Diseases of the skin and subcutaneous tissue_both sexes_age_all ages - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_under_1: title: Diseases of the skin and subcutaneous tissue_both sexes_age_under_1 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_1_4: title: Diseases of the skin and subcutaneous tissue_both sexes_age_1-4 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_5_9: title: Diseases of the skin and subcutaneous tissue_both sexes_age_5-9 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_10_14: title: Diseases of the skin and subcutaneous tissue_both sexes_age_10-14 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_15_19: title: Diseases of the skin and subcutaneous tissue_both sexes_age_15-19 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_20_24: title: Diseases of the skin and subcutaneous tissue_both sexes_age_20-24 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_25_29: title: Diseases of the skin and subcutaneous tissue_both sexes_age_25-29 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_30_34: title: Diseases of the skin and subcutaneous tissue_both sexes_age_30-34 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_35_39: title: Diseases of the skin and subcutaneous tissue_both sexes_age_35-39 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_40_44: title: Diseases of the skin and subcutaneous tissue_both sexes_age_40-44 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_45_49: title: Diseases of the skin and subcutaneous tissue_both sexes_age_45-49 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_50_54: title: Diseases of the skin and subcutaneous tissue_both sexes_age_50-54 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_55_59: title: Diseases of the skin and subcutaneous tissue_both sexes_age_55-59 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_60_64: title: Diseases of the skin and subcutaneous tissue_both sexes_age_60-64 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_65_69: title: Diseases of the skin and subcutaneous tissue_both sexes_age_65-69 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_70_74: title: Diseases of the skin and subcutaneous tissue_both sexes_age_70-74 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_75_79: title: Diseases of the skin and subcutaneous tissue_both sexes_age_75-79 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_80_84: title: Diseases of the skin and subcutaneous tissue_both sexes_age_80-84 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_85_89: title: Diseases of the skin and subcutaneous tissue_both sexes_age_85-89 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_90_94: title: Diseases of the skin and subcutaneous tissue_both sexes_age_90-94 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_95_99: title: Diseases of the skin and subcutaneous tissue_both sexes_age_95-99 - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_skin_and_subcutaneous_tissue_both_sexes_age_100_and_over: title: Diseases of the skin and subcutaneous tissue_both sexes_age_100_and_over - description: The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. - Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. unit: per 100 000 + description: |- + The mortality rate from diseases of the skin and subcutaneous tissue per 100 000 people in the population. Diseases of the skin and subcutaneous tissue have the ICD death codes 680*-709*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_all_ages: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_all ages - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_under_1: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_under_1 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_1_4: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_1-4 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_5_9: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_5-9 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_10_14: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_10-14 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_15_19: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_15-19 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_20_24: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_20-24 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_25_29: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_25-29 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_30_34: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_30-34 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_35_39: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_35-39 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_40_44: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_40-44 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_45_49: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_45-49 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_50_54: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_50-54 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_55_59: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_55-59 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_60_64: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_60-64 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_65_69: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_65-69 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_70_74: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_70-74 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_75_79: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_75-79 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_80_84: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_80-84 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_85_89: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_85-89 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_90_94: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_90-94 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_95_99: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_95-99 - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. diseases_of_the_musculoskeletal_system_and_connective_tissue_both_sexes_age_100_and_over: title: Diseases of the musculoskeletal system and connective tissue_both sexes_age_100_and_over - description: The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people - in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. unit: per 100 000 + description: |- + The mortality rate from diseases of the musculoskeletal system and connective tissue per 100 000 people in the population. Diseases of the musculoskeletal system and connective tissue have the ICD death codes 710*-739*. congenital_anomalies_both_sexes_age_all_ages: title: Congenital anomalies_both sexes_age_all ages - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_under_1: title: Congenital anomalies_both sexes_age_under_1 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_1_4: title: Congenital anomalies_both sexes_age_1-4 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_5_9: title: Congenital anomalies_both sexes_age_5-9 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_10_14: title: Congenital anomalies_both sexes_age_10-14 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_15_19: title: Congenital anomalies_both sexes_age_15-19 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_20_24: title: Congenital anomalies_both sexes_age_20-24 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_25_29: title: Congenital anomalies_both sexes_age_25-29 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_30_34: title: Congenital anomalies_both sexes_age_30-34 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_35_39: title: Congenital anomalies_both sexes_age_35-39 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_40_44: title: Congenital anomalies_both sexes_age_40-44 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_45_49: title: Congenital anomalies_both sexes_age_45-49 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_50_54: title: Congenital anomalies_both sexes_age_50-54 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_55_59: title: Congenital anomalies_both sexes_age_55-59 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_60_64: title: Congenital anomalies_both sexes_age_60-64 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_65_69: title: Congenital anomalies_both sexes_age_65-69 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_70_74: title: Congenital anomalies_both sexes_age_70-74 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_75_79: title: Congenital anomalies_both sexes_age_75-79 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_80_84: title: Congenital anomalies_both sexes_age_80-84 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_85_89: title: Congenital anomalies_both sexes_age_85-89 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_90_94: title: Congenital anomalies_both sexes_age_90-94 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_95_99: title: Congenital anomalies_both sexes_age_95-99 - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. congenital_anomalies_both_sexes_age_100_and_over: title: Congenital anomalies_both sexes_age_100_and_over - description: The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies - have the ICD death codes 740*-759*. unit: per 100 000 + description: |- + The mortality rate from congenital anomalies per 100 000 people in the population. Congenital anomalies have the ICD death codes 740*-759*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_all_ages: title: Certain conditions originating in the perinatal period_both sexes_age_all ages - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_under_1: title: Certain conditions originating in the perinatal period_both sexes_age_under_1 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_1_4: title: Certain conditions originating in the perinatal period_both sexes_age_1-4 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_5_9: title: Certain conditions originating in the perinatal period_both sexes_age_5-9 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_10_14: title: Certain conditions originating in the perinatal period_both sexes_age_10-14 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_15_19: title: Certain conditions originating in the perinatal period_both sexes_age_15-19 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_20_24: title: Certain conditions originating in the perinatal period_both sexes_age_20-24 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_25_29: title: Certain conditions originating in the perinatal period_both sexes_age_25-29 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_30_34: title: Certain conditions originating in the perinatal period_both sexes_age_30-34 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_35_39: title: Certain conditions originating in the perinatal period_both sexes_age_35-39 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_40_44: title: Certain conditions originating in the perinatal period_both sexes_age_40-44 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_45_49: title: Certain conditions originating in the perinatal period_both sexes_age_45-49 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_50_54: title: Certain conditions originating in the perinatal period_both sexes_age_50-54 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_55_59: title: Certain conditions originating in the perinatal period_both sexes_age_55-59 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_60_64: title: Certain conditions originating in the perinatal period_both sexes_age_60-64 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_65_69: title: Certain conditions originating in the perinatal period_both sexes_age_65-69 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_70_74: title: Certain conditions originating in the perinatal period_both sexes_age_70-74 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_75_79: title: Certain conditions originating in the perinatal period_both sexes_age_75-79 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_80_84: title: Certain conditions originating in the perinatal period_both sexes_age_80-84 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_85_89: title: Certain conditions originating in the perinatal period_both sexes_age_85-89 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_90_94: title: Certain conditions originating in the perinatal period_both sexes_age_90-94 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_95_99: title: Certain conditions originating in the perinatal period_both sexes_age_95-99 - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. certain_conditions_originating_in_the_perinatal_period_both_sexes_age_100_and_over: title: Certain conditions originating in the perinatal period_both sexes_age_100_and_over - description: The mortality rate from certain conditions originating in the perinatal period per 100 000 people in - the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. unit: per 100 000 + description: |- + The mortality rate from certain conditions originating in the perinatal period per 100 000 people in the population. Certain conditions originating in the perinatal period have the ICD death codes 760*-779*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_all_ages: title: Symptoms signs and ill-defined conditions_both sexes_age_all ages - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_under_1: title: Symptoms signs and ill-defined conditions_both sexes_age_under_1 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_1_4: title: Symptoms signs and ill-defined conditions_both sexes_age_1-4 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_5_9: title: Symptoms signs and ill-defined conditions_both sexes_age_5-9 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_10_14: title: Symptoms signs and ill-defined conditions_both sexes_age_10-14 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_15_19: title: Symptoms signs and ill-defined conditions_both sexes_age_15-19 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_20_24: title: Symptoms signs and ill-defined conditions_both sexes_age_20-24 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_25_29: title: Symptoms signs and ill-defined conditions_both sexes_age_25-29 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_30_34: title: Symptoms signs and ill-defined conditions_both sexes_age_30-34 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_35_39: title: Symptoms signs and ill-defined conditions_both sexes_age_35-39 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_40_44: title: Symptoms signs and ill-defined conditions_both sexes_age_40-44 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_45_49: title: Symptoms signs and ill-defined conditions_both sexes_age_45-49 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_50_54: title: Symptoms signs and ill-defined conditions_both sexes_age_50-54 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_55_59: title: Symptoms signs and ill-defined conditions_both sexes_age_55-59 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_60_64: title: Symptoms signs and ill-defined conditions_both sexes_age_60-64 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_65_69: title: Symptoms signs and ill-defined conditions_both sexes_age_65-69 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_70_74: title: Symptoms signs and ill-defined conditions_both sexes_age_70-74 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_75_79: title: Symptoms signs and ill-defined conditions_both sexes_age_75-79 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_80_84: title: Symptoms signs and ill-defined conditions_both sexes_age_80-84 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_85_89: title: Symptoms signs and ill-defined conditions_both sexes_age_85-89 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_90_94: title: Symptoms signs and ill-defined conditions_both sexes_age_90-94 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_95_99: title: Symptoms signs and ill-defined conditions_both sexes_age_95-99 - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. symptoms_signs_and_ill_defined_conditions_both_sexes_age_100_and_over: title: Symptoms signs and ill-defined conditions_both sexes_age_100_and_over - description: The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. - Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. unit: per 100 000 + description: |- + The mortality rate from symptoms, signs and ill-defined conditions per 100 000 people in the population. Symptoms, signs and ill-defined conditions have the ICD death codes 780*-799*. injury_and_poisoning_both_sexes_age_all_ages: title: Injury and poisoning_both sexes_age_all ages - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_under_1: title: Injury and poisoning_both sexes_age_under_1 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_1_4: title: Injury and poisoning_both sexes_age_1-4 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_5_9: title: Injury and poisoning_both sexes_age_5-9 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_10_14: title: Injury and poisoning_both sexes_age_10-14 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_15_19: title: Injury and poisoning_both sexes_age_15-19 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_20_24: title: Injury and poisoning_both sexes_age_20-24 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_25_29: title: Injury and poisoning_both sexes_age_25-29 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_30_34: title: Injury and poisoning_both sexes_age_30-34 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_35_39: title: Injury and poisoning_both sexes_age_35-39 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_40_44: title: Injury and poisoning_both sexes_age_40-44 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_45_49: title: Injury and poisoning_both sexes_age_45-49 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_50_54: title: Injury and poisoning_both sexes_age_50-54 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_55_59: title: Injury and poisoning_both sexes_age_55-59 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_60_64: title: Injury and poisoning_both sexes_age_60-64 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_65_69: title: Injury and poisoning_both sexes_age_65-69 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_70_74: title: Injury and poisoning_both sexes_age_70-74 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_75_79: title: Injury and poisoning_both sexes_age_75-79 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_80_84: title: Injury and poisoning_both sexes_age_80-84 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_85_89: title: Injury and poisoning_both sexes_age_85-89 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_90_94: title: Injury and poisoning_both sexes_age_90-94 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_95_99: title: Injury and poisoning_both sexes_age_95-99 - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. injury_and_poisoning_both_sexes_age_100_and_over: title: Injury and poisoning_both sexes_age_100_and_over - description: The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning - have the ICD death codes 800*-999*. unit: per 100 000 + description: |- + The mortality rate from injury and poisoning per 100 000 people in the population. Injury and poisoning have the ICD death codes 800*-999*. diff --git a/etl/steps/data/grapher/fasttrack/latest/historical_france_mortality_cause.py b/etl/steps/data/grapher/fasttrack/latest/historical_france_mortality_cause.py index b54066c948a..6f8672c6cb1 100644 --- a/etl/steps/data/grapher/fasttrack/latest/historical_france_mortality_cause.py +++ b/etl/steps/data/grapher/fasttrack/latest/historical_france_mortality_cause.py @@ -1,10 +1,9 @@ import pandas as pd -from owid import catalog -from etl.helpers import PathFinder, create_dataset +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot -P = PathFinder(__file__) +paths = PathFinder(__file__) def run(dest_dir: str) -> None: @@ -12,11 +11,30 @@ def run(dest_dir: str) -> None: snap = Snapshot("fasttrack/latest/historical_france_mortality_cause.csv") # load data - data = pd.read_csv(snap.path) + tb = snap.read_csv() - # create empty dataframe and table - tb = catalog.Table(data, short_name=P.short_name) + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) # add table, update metadata from *.meta.yml and save - ds = create_dataset(dest_dir, tables=[tb.set_index(["country", "year"])], default_metadata=snap.metadata) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/latest/infant_mortality_vaccination_shattock.py b/etl/steps/data/grapher/fasttrack/latest/infant_mortality_vaccination_shattock.py index 839a3b24e99..ca5755007a5 100644 --- a/etl/steps/data/grapher/fasttrack/latest/infant_mortality_vaccination_shattock.py +++ b/etl/steps/data/grapher/fasttrack/latest/infant_mortality_vaccination_shattock.py @@ -1,3 +1,5 @@ +import pandas as pd + from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot @@ -11,8 +13,20 @@ def run(dest_dir: str) -> None: # load data tb = snap.read_csv() + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) + # add table, update metadata from *.meta.yml and save - ds = create_dataset(dest_dir, tables=[tb.set_index(["country", "year"])], default_metadata=snap.metadata) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) # override metadata if necessary meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") @@ -20,3 +34,7 @@ def run(dest_dir: str) -> None: ds.update_metadata(meta_path) ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/latest/lead_paint_regulation_who.meta.yml b/etl/steps/data/grapher/fasttrack/latest/lead_paint_regulation_who.meta.yml index b84e3a73d99..9467e2b5fae 100644 --- a/etl/steps/data/grapher/fasttrack/latest/lead_paint_regulation_who.meta.yml +++ b/etl/steps/data/grapher/fasttrack/latest/lead_paint_regulation_who.meta.yml @@ -1,28 +1,16 @@ dataset: - namespace: fasttrack - version: latest - short_name: lead_paint_regulation_who title: Lead paint regulations (WHO, 2023) description: |- The WHO collects data on which countries have legally-binding controls on lead paint. It sources this data from surveys conducted by WHO and UNEP of national authorities. The World Health Organization (WHO) tracks the introduction of legally-binding controls on lead concentrations in paint. Paint is a main contributor to harmful lead exposure. The stringency of controls on lead paint can vary by country. Maximum concentrations of lead can differ, and may only apply to particular types of paint (for example, products used in households). - sources: - - name: Lead paint regulations (WHO, 2023) - published_by: World Health Organization (WHO) - publication_year: 2023 - url: https://www.who.int/data/gho/data/themes/topics/indicator-groups/legally-binding-controls-on-lead-paint + licenses: + - {} tables: lead_paint_regulation_who: variables: lead_paint_regulation: title: lead_paint_regulation unit: '' - description: Notes whether each country has adopted legally-binding laws, - regulations, standards and/or procedures to control the production, import, - export, sale and use of lead paints. - sources: - - name: Lead paint regulations (WHO, 2023) - published_by: World Health Organization (WHO) - publication_year: 2023 - url: https://www.who.int/data/gho/data/themes/topics/indicator-groups/legally-binding-controls-on-lead-paint + description: |- + Notes whether each country has adopted legally-binding laws, regulations, standards and/or procedures to control the production, import, export, sale and use of lead paints. diff --git a/etl/steps/data/grapher/fasttrack/latest/lead_paint_regulation_who.py b/etl/steps/data/grapher/fasttrack/latest/lead_paint_regulation_who.py index 1f0c2a1a5e8..c44f9d8ac86 100644 --- a/etl/steps/data/grapher/fasttrack/latest/lead_paint_regulation_who.py +++ b/etl/steps/data/grapher/fasttrack/latest/lead_paint_regulation_who.py @@ -1,19 +1,40 @@ import pandas as pd -from owid import catalog -from etl.helpers import PathFinder, create_dataset +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot -P = PathFinder(__file__) +paths = PathFinder(__file__) def run(dest_dir: str) -> None: # load snapshot - data = pd.read_csv(Snapshot("fasttrack/latest/lead_paint_regulation_who.csv").path) + snap = Snapshot("fasttrack/latest/lead_paint_regulation_who.csv") - # create empty dataframe and table - tb = catalog.Table(data, short_name=P.short_name) + # load data + tb = snap.read_csv() + + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) # add table, update metadata from *.meta.yml and save - ds = create_dataset(dest_dir, tables=[tb]) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/latest/lives_saved_vaccination_who.py b/etl/steps/data/grapher/fasttrack/latest/lives_saved_vaccination_who.py index 17ec65f21ba..3fcf6026ee9 100644 --- a/etl/steps/data/grapher/fasttrack/latest/lives_saved_vaccination_who.py +++ b/etl/steps/data/grapher/fasttrack/latest/lives_saved_vaccination_who.py @@ -1,3 +1,5 @@ +import pandas as pd + from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot @@ -11,8 +13,20 @@ def run(dest_dir: str) -> None: # load data tb = snap.read_csv() + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) + # add table, update metadata from *.meta.yml and save - ds = create_dataset(dest_dir, tables=[tb.set_index(["country", "year"])], default_metadata=snap.metadata) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) # override metadata if necessary meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") @@ -20,3 +34,7 @@ def run(dest_dir: str) -> None: ds.update_metadata(meta_path) ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/latest/nuclear_warhead_inventories.meta.yml b/etl/steps/data/grapher/fasttrack/latest/nuclear_warhead_inventories.meta.yml new file mode 100644 index 00000000000..182af2197ab --- /dev/null +++ b/etl/steps/data/grapher/fasttrack/latest/nuclear_warhead_inventories.meta.yml @@ -0,0 +1,46 @@ +dataset: + title: Nuclear warhead inventories – Federation of American Scientists + description: |- + This dataset provides information on the nuclear warhead inventories by the nuclear powers, using data from the Federation of American Scientists, prepared by Hans M. Kristensen, Matt Korda, and Robert Norris. + + You can download the code and complete dataset, including supplementary variables, from GitHub: https://github.com/owid/notebooks/tree/main/BastianHerre/nuclear_weapons + Publisher source: None + licenses: + - {} +tables: + nuclear_warhead_inventories: + variables: + nuclear_weapons_depl_nonstrat: + title: nuclear_weapons_depl_nonstrat + unit: '' + description: |- + The variable denotes the estimated number of deployed nonstrategic nuclear warheads. + + Deployed warheads are those on ballistic missiles or bomber bases. + + Nonstrategic or tactical warheads are those for use on the battlefield. + nuclear_weapons_depl_strat: + title: nuclear_weapons_depl_strat + unit: '' + description: |- + The variable denotes the estimated number of deployed strategic nuclear warheads. + + Deployed warheads are those on ballistic missiles or bomber bases. + + Strategic warheads are those for use away from the battlefield, such as against military bases, arms industries, or infrastructure. + nuclear_weapons_inventory: + title: nuclear_weapons_inventory + unit: '' + description: |- + The variable denotes the estimated number of all nuclear warheads, be they deployed strategic, deployed nonstrategic, nondeployed, or retired. + nuclear_weapons_reserve_nondepl: + title: nuclear_weapons_reserve_nondepl + unit: '' + description: |- + The variable denotes the estimated number of nondeployed nuclear warheads. + + Nondeployed or reserve warheads are those not on ballistic missiles or bomber bases. + nuclear_weapons_retired: + title: nuclear_weapons_retired + unit: '' + description: The variable denotes the estimated number of retired nuclear warheads queued for dismantlement. diff --git a/etl/steps/data/grapher/fasttrack/latest/nuclear_warhead_inventories.py b/etl/steps/data/grapher/fasttrack/latest/nuclear_warhead_inventories.py new file mode 100644 index 00000000000..54f9902ad56 --- /dev/null +++ b/etl/steps/data/grapher/fasttrack/latest/nuclear_warhead_inventories.py @@ -0,0 +1,40 @@ +import pandas as pd + +from etl.helpers import PathFinder, create_dataset, get_metadata_path +from etl.snapshot import Snapshot + +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # load snapshot + snap = Snapshot("fasttrack/latest/nuclear_warhead_inventories.csv") + + # load data + tb = snap.read_csv() + + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) + + # add table, update metadata from *.meta.yml and save + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/latest/nuclear_warhead_stockpiles.meta.yml b/etl/steps/data/grapher/fasttrack/latest/nuclear_warhead_stockpiles.meta.yml new file mode 100644 index 00000000000..671f7b2dd74 --- /dev/null +++ b/etl/steps/data/grapher/fasttrack/latest/nuclear_warhead_stockpiles.meta.yml @@ -0,0 +1,21 @@ +dataset: + title: Nuclear warhead stockpiles – Federation of American Scientists + description: |- + This dataset provides information on the number of stockpiled nuclear warheads by the nuclear powers, using data from the Federation of American Scientists, prepared by Hans M. Kristensen, Matt Korda, Eliana Reynolds, and Robert Norris. + + You can download the code and complete dataset, including supplementary variables, from GitHub: https://github.com/owid/notebooks/tree/main/BastianHerre/nuclear_weapons + Publisher source: None + licenses: + - {} +tables: + nuclear_warhead_stockpiles: + variables: + nuclear_weapons_stockpile: + title: nuclear_weapons_stockpile + unit: '' + description: |- + The variable denotes the estimated number of nuclear warheads in the stockpiles of the nuclear powers. + + Stockpiles include warheads assigned to military forces, but exclude retired warheads queued for dismantlement. + + Retired warheads are only included in the global total. diff --git a/etl/steps/data/grapher/fasttrack/latest/nuclear_warhead_stockpiles.py b/etl/steps/data/grapher/fasttrack/latest/nuclear_warhead_stockpiles.py new file mode 100644 index 00000000000..d27ca9006ec --- /dev/null +++ b/etl/steps/data/grapher/fasttrack/latest/nuclear_warhead_stockpiles.py @@ -0,0 +1,40 @@ +import pandas as pd + +from etl.helpers import PathFinder, create_dataset, get_metadata_path +from etl.snapshot import Snapshot + +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # load snapshot + snap = Snapshot("fasttrack/latest/nuclear_warhead_stockpiles.csv") + + # load data + tb = snap.read_csv() + + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) + + # add table, update metadata from *.meta.yml and save + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/latest/pain_hours_hen_systems.meta.yml b/etl/steps/data/grapher/fasttrack/latest/pain_hours_hen_systems.meta.yml index f5b32980f62..83049793925 100644 --- a/etl/steps/data/grapher/fasttrack/latest/pain_hours_hen_systems.meta.yml +++ b/etl/steps/data/grapher/fasttrack/latest/pain_hours_hen_systems.meta.yml @@ -1,69 +1,68 @@ dataset: - sources: - - name: Welfare Footprint based on Schuck-Paim and Alonso (2021) - published_by: Welfare Footprint - publication_year: 2021 - url: https://welfarefootprint.org/research-projects/laying-hens/ + title: Pain hours of hen systems (Welfare Footprint) + description: '' + licenses: + - {} tables: pain_hours_hen_systems: variables: total_hours_of_pain: title: Total hours of pain - short_unit: hours unit: hours - description: The total number of hours an average hen will spend in pain, regardless of the intensity. - display: - numDecimalPlaces: 0 - sources: - - name: Welfare Footprint based on Schuck-Paim and Alonso (2021) - published_by: Welfare Footprint - publication_year: 2021 - url: https://welfarefootprint.org/research-projects/laying-hens/ - excrutiating_pain: - title: Excrutiating pain short_unit: hours - unit: hours - description: The number of hours an average hen will spend in excrutiating pain. display: - numDecimalPlaces: 2 - sources: - - name: Welfare Footprint based on Schuck-Paim and Alonso (2021) - published_by: Welfare Footprint - publication_year: 2021 - url: https://welfarefootprint.org/research-projects/laying-hens/ - disabling_pain: - title: Disabling pain - short_unit: hours + numDecimalPlaces: 0.0 + description: The total number of hours an average hen will spend in pain, regardless of the intensity. + excrutiating_pain_in_hours: + title: Excrutiating pain in hours unit: hours - description: The number of hours an average hen will spend in disabling pain. - display: - numDecimalPlaces: 0 - sources: - - name: Welfare Footprint based on Schuck-Paim and Alonso (2021) - published_by: Welfare Footprint - publication_year: 2021 - url: https://welfarefootprint.org/research-projects/laying-hens/ - hurtful_pain: - title: Hurtful pain short_unit: hours + display: + numDecimalPlaces: 2.0 + description: The number of hours an average hen will spend in excrutiating pain. + disabling_pain_in_hours: + title: Disabling pain in hours unit: hours - description: The number of hours an average hen will spend in hurtful pain. + short_unit: hours display: - numDecimalPlaces: 0 - sources: - - name: Welfare Footprint based on Schuck-Paim and Alonso (2021) - published_by: Welfare Footprint - publication_year: 2021 - url: https://welfarefootprint.org/research-projects/laying-hens/ - annoying_pain: - title: Annoying pain + numDecimalPlaces: 0.0 + description: The number of hours an average hen will spend in disabling pain. + hurtful_pain_in_hours: + title: Hurtful pain in hours + unit: hours short_unit: hours + display: + numDecimalPlaces: 0.0 + description: The number of hours an average hen will spend in hurtful pain. + annoying_pain_in_hours: + title: Annoying pain in hours unit: hours - description: The number of hours an average hen will spend in annoying pain. + short_unit: hours display: - numDecimalPlaces: 0 - sources: - - name: Welfare Footprint based on Schuck-Paim and Alonso (2021) - published_by: Welfare Footprint - publication_year: 2021 - url: https://welfarefootprint.org/research-projects/laying-hens/ + numDecimalPlaces: 0.0 + description: The number of hours an average hen will spend in annoying pain. + total_days_of_pain: + title: Total days of pain + unit: days + short_unit: days + description: The total number of days an average hen will spend in pain, regardless of the intensity. + excrutiating_pain_in_days: + title: Excrutiating pain in days + unit: days + short_unit: days + description: The number of days an average hen will spend in excrutiating pain. + disabling_pain_in_days: + title: Disabling pain in days + unit: days + short_unit: days + description: The number of days an average hen will spend in disabling pain. + hurtful_pain_in_days: + title: Hurtful pain in days + unit: days + short_unit: days + description: The number of days an average hen will spend in hurtful pain. + annoying_pain_in_days: + title: Annoying pain in days + unit: days + short_unit: days + description: The number of days an average hen will spend in annoying pain. diff --git a/etl/steps/data/grapher/fasttrack/latest/pain_hours_hen_systems.py b/etl/steps/data/grapher/fasttrack/latest/pain_hours_hen_systems.py index 2a12110de76..b7bce222145 100644 --- a/etl/steps/data/grapher/fasttrack/latest/pain_hours_hen_systems.py +++ b/etl/steps/data/grapher/fasttrack/latest/pain_hours_hen_systems.py @@ -1,10 +1,9 @@ import pandas as pd -from owid import catalog -from etl.helpers import PathFinder, create_dataset +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot -P = PathFinder(__file__) +paths = PathFinder(__file__) def run(dest_dir: str) -> None: @@ -12,11 +11,30 @@ def run(dest_dir: str) -> None: snap = Snapshot("fasttrack/latest/pain_hours_hen_systems.csv") # load data - data = pd.read_csv(snap.path) + tb = snap.read_csv() - # create empty dataframe and table - tb = catalog.Table(data, short_name=P.short_name) + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) # add table, update metadata from *.meta.yml and save ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/latest/plastic_waste_meijer_2021.meta.yml b/etl/steps/data/grapher/fasttrack/latest/plastic_waste_meijer_2021.meta.yml index d7c1587f391..a3e10290266 100644 --- a/etl/steps/data/grapher/fasttrack/latest/plastic_waste_meijer_2021.meta.yml +++ b/etl/steps/data/grapher/fasttrack/latest/plastic_waste_meijer_2021.meta.yml @@ -3,12 +3,6 @@ dataset: description: '' licenses: - {} - sources: - - name: Plastic ocean waste and pollution (Meijer et al. 2021) - url: https://www.science.org/doi/10.1126/sciadv.aaz5803 - publication_year: '2021' - published_by: Meijer, L. J., Van Emmerik, T., Van Der Ent, R., Schmidt, C., & Lebreton, L. (2021). More than 1000 rivers - account for 80% of global riverine plastic emissions into the ocean. Science Advances, 7(18), eaaz5803. tables: plastic_waste_meijer_2021: variables: diff --git a/etl/steps/data/grapher/fasttrack/latest/plastic_waste_meijer_2021.py b/etl/steps/data/grapher/fasttrack/latest/plastic_waste_meijer_2021.py index 0c199a44836..af0c97aade1 100644 --- a/etl/steps/data/grapher/fasttrack/latest/plastic_waste_meijer_2021.py +++ b/etl/steps/data/grapher/fasttrack/latest/plastic_waste_meijer_2021.py @@ -1,7 +1,6 @@ import pandas as pd -from owid import catalog -from etl.helpers import PathFinder, create_dataset +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot paths = PathFinder(__file__) @@ -12,11 +11,30 @@ def run(dest_dir: str) -> None: snap = Snapshot("fasttrack/latest/plastic_waste_meijer_2021.csv") # load data - data = pd.read_csv(snap.path) + tb = snap.read_csv() - # create empty dataframe and table - tb = catalog.Table(data, short_name=paths.short_name) + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) # add table, update metadata from *.meta.yml and save - ds = create_dataset(dest_dir, tables=[tb.set_index(["country", "year"])], default_metadata=snap.metadata) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/latest/sentience_institute.meta.yml b/etl/steps/data/grapher/fasttrack/latest/sentience_institute.meta.yml index 737a84df55d..269bf760b42 100644 --- a/etl/steps/data/grapher/fasttrack/latest/sentience_institute.meta.yml +++ b/etl/steps/data/grapher/fasttrack/latest/sentience_institute.meta.yml @@ -3,11 +3,6 @@ dataset: description: '' licenses: - {} - sources: - - name: 'Sentience Institute. Animals, Food, and Technology (AFT) Survey: 2021 Update.' - url: https://www.sentienceinstitute.org/aft-survey-2021 - publication_year: '2021' - published_by: Sentience Institute tables: sentience_institute: variables: diff --git a/etl/steps/data/grapher/fasttrack/latest/sentience_institute.py b/etl/steps/data/grapher/fasttrack/latest/sentience_institute.py index 687cd4deb83..8a413131d69 100644 --- a/etl/steps/data/grapher/fasttrack/latest/sentience_institute.py +++ b/etl/steps/data/grapher/fasttrack/latest/sentience_institute.py @@ -1,7 +1,6 @@ import pandas as pd -from owid import catalog -from etl.helpers import PathFinder, create_dataset +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot paths = PathFinder(__file__) @@ -12,11 +11,30 @@ def run(dest_dir: str) -> None: snap = Snapshot("fasttrack/latest/sentience_institute.csv") # load data - data = pd.read_csv(snap.path) + tb = snap.read_csv() - # create empty dataframe and table - tb = catalog.Table(data, short_name=paths.short_name) + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) # add table, update metadata from *.meta.yml and save - ds = create_dataset(dest_dir, tables=[tb.set_index(["country", "year"])], default_metadata=snap.metadata) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/latest/treatment_gap_anxiety_disorders_world_mental_health_surveys.meta.yml b/etl/steps/data/grapher/fasttrack/latest/treatment_gap_anxiety_disorders_world_mental_health_surveys.meta.yml index a85f3078d93..ac639754121 100644 --- a/etl/steps/data/grapher/fasttrack/latest/treatment_gap_anxiety_disorders_world_mental_health_surveys.meta.yml +++ b/etl/steps/data/grapher/fasttrack/latest/treatment_gap_anxiety_disorders_world_mental_health_surveys.meta.yml @@ -1,78 +1,40 @@ dataset: - namespace: fasttrack - version: latest - short_name: treatment_gap_anxiety_disorders_world_mental_health_surveys - title: Treatment gap for anxiety disorders - World Mental Health Surveys - Alonso - et al. 2017 - description: 'This dataset comes from the World Mental Health surveys, which conducted - national studies in 21 countries, using validated structured interviews to survey - members of the general population about symptoms of mental illnesses they had - in the past 12 months and their lifetime so far. The source describes the dataset: - "Data came from 24 community epidemiological surveys administered in 21 countries - as part of the WMH surveys (Kessler & Ustun, 2004). These included 12 surveys - carried out in high-income countries, 6 surveys in upper-middle-income countries - and 6 in low or lower-middle income countries (see table 1). The majority of surveys - were based on nationally representative household samples. Three were representative - of urban areas in their countries (Colombia, Mexico, and Peru). Three were representative - of selected regions in their countries (Japan, Nigeria, and Murcia, Spain). Four - were representative of selected Metropolitan Areas (Sao Paulo, Brazil; Medellin, - Colombia; and Beijing-Shanghai and Shenzhen in the People’s Republic of China - (PRC)). Trained lay interviewers conducted face-to-face interviews with respondents, - aged 18 years and over. The interviews took place within the households of the - respondents. To reduce respondent burden, the interview was divided into two parts. - Part I assessed core mental disorders and was administered to all respondents. - Part II, which assessed additional disorders and correlates, was administered - to all Part I respondents who met lifetime criteria for any disorder plus a probability - subsample of other Part I respondents. Part II data, the focus of this report, - were weighted by the inverse of their probabilities of selection into Part II - and additionally weighted to adjust samples to match population distributions - on the cross-classification of key socio-demographic and geographic variables. - Further details about WMH sampling and weighting are available elsewhere(Heeringa - et al., 2008). Response rates ranged between 45.9% and 97.2% and had a weighted - average of 70.1% across all surveys."' - sources: - - name: Treatment gap for anxiety disorders - World Mental Health Surveys - Alonso - et al. 2017 - published_by: Alonso et al. (2017) - description: Data comes from Community surveys of the general population - publication_year: 2017 - date_accessed: 2023-05-11 - url: https://pubmed.ncbi.nlm.nih.gov/29356216/ + title: Treatment gap for anxiety disorders - World Mental Health Surveys - Alonso et al. 2017 + description: |- + This dataset comes from the World Mental Health surveys, which conducted national studies in 21 countries, using validated structured interviews to survey members of the general population about symptoms of mental illnesses they had in the past 12 months and their lifetime so far. The source describes the dataset: "Data came from 24 community epidemiological surveys administered in 21 countries as part of the WMH surveys (Kessler & Ustun, 2004). These included 12 surveys carried out in high-income countries, 6 surveys in upper-middle-income countries and 6 in low or lower-middle income countries (see table 1). The majority of surveys were based on nationally representative household samples. Three were representative of urban areas in their countries (Colombia, Mexico, and Peru). Three were representative of selected regions in their countries (Japan, Nigeria, and Murcia, Spain). Four were representative of selected Metropolitan Areas (Sao Paulo, Brazil; Medellin, Colombia; and Beijing-Shanghai and Shenzhen in the People’s Republic of China (PRC)). Trained lay interviewers conducted face-to-face interviews with respondents, aged 18 years and over. The interviews took place within the households of the respondents. To reduce respondent burden, the interview was divided into two parts. Part I assessed core mental disorders and was administered to all respondents. Part II, which assessed additional disorders and correlates, was administered to all Part I respondents who met lifetime criteria for any disorder plus a probability subsample of other Part I respondents. Part II data, the focus of this report, were weighted by the inverse of their probabilities of selection into Part II and additionally weighted to adjust samples to match population distributions on the cross-classification of key socio-demographic and geographic variables. Further details about WMH sampling and weighting are available elsewhere(Heeringa et al., 2008). Response rates ranged between 45.9% and 97.2% and had a weighted average of 70.1% across all surveys." + licenses: + - name: Made freely available by authors + url: https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6008788/ tables: treatment_gap_anxiety_disorders_world_mental_health_surveys: variables: _12_month_anxiety_disorder: title: 12 month anxiety disorder - short_unit: '%' unit: '%' - description: Share who met the criteria for having an anxiety disorder in - the past 12 months. The Composite International Diagnostic Interview (CIDI) - was used to interview participants about symptoms of anxiety. + short_unit: '%' + description: |- + Share who met the criteria for having an anxiety disorder in the past 12 months. The Composite International Diagnostic Interview (CIDI) was used to interview participants about symptoms of anxiety. any_treatment__conditional: title: Any treatment, conditional - short_unit: '%' unit: '%' - description: Share who received any treatment, among those who met the criteria - for an anxiety disorder in the past 12 months. + short_unit: '%' + description: Share who received any treatment, among those who met the criteria for an anxiety disorder in the past + 12 months. possibly_adequate_treatment__conditional: title: Possibly adequate treatment, conditional - short_unit: '%' unit: '%' - description: Share who received potentially adequate treatment, among those - who met the criteria for an anxiety disorder in the past 12 months. "Potentially - adequate treatment" was defined as pharmacological medication, psychotherapy, - or complementary alternative medicine. + short_unit: '%' + description: |- + Share who received potentially adequate treatment, among those who met the criteria for an anxiety disorder in the past 12 months. "Potentially adequate treatment" was defined as pharmacological medication, psychotherapy, or complementary alternative medicine. any_treatment: title: Any treatment - short_unit: '%' unit: '%' - description: Share who met the criteria for having an anxiety disorder in - the past 12 months and received any treatment for it. + short_unit: '%' + description: Share who met the criteria for having an anxiety disorder in the past 12 months and received any treatment + for it. possibly_adequate_treatment: title: Possibly adequate treatment - short_unit: '%' unit: '%' - description: Share who met the criteria for having an anxiety disorder in - the past 12 months and received potentially adequate treatment for it. "Potentially - adequate treatment" was defined as pharmacological medication, psychotherapy, - or complementary alternative medicine. + short_unit: '%' + description: |- + Share who met the criteria for having an anxiety disorder in the past 12 months and received potentially adequate treatment for it. "Potentially adequate treatment" was defined as pharmacological medication, psychotherapy, or complementary alternative medicine. diff --git a/etl/steps/data/grapher/fasttrack/latest/treatment_gap_anxiety_disorders_world_mental_health_surveys.py b/etl/steps/data/grapher/fasttrack/latest/treatment_gap_anxiety_disorders_world_mental_health_surveys.py index 667ddeb4ecd..e6fe95e7448 100644 --- a/etl/steps/data/grapher/fasttrack/latest/treatment_gap_anxiety_disorders_world_mental_health_surveys.py +++ b/etl/steps/data/grapher/fasttrack/latest/treatment_gap_anxiety_disorders_world_mental_health_surveys.py @@ -1,21 +1,40 @@ import pandas as pd -from owid import catalog -from etl.helpers import PathFinder, create_dataset +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot -P = PathFinder(__file__) +paths = PathFinder(__file__) def run(dest_dir: str) -> None: # load snapshot - data = pd.read_csv( - Snapshot("fasttrack/latest/treatment_gap_anxiety_disorders_world_mental_health_surveys.csv").path - ) + snap = Snapshot("fasttrack/latest/treatment_gap_anxiety_disorders_world_mental_health_surveys.csv") - # create empty dataframe and table - tb = catalog.Table(data, short_name=P.short_name) + # load data + tb = snap.read_csv() + + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) # add table, update metadata from *.meta.yml and save - ds = create_dataset(dest_dir, tables=[tb]) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/fasttrack/latest/whm_treatment_gap_anxiety_disorders.meta.yml b/etl/steps/data/grapher/fasttrack/latest/whm_treatment_gap_anxiety_disorders.meta.yml index a5e2e7d9488..8b91dd1f1ae 100644 --- a/etl/steps/data/grapher/fasttrack/latest/whm_treatment_gap_anxiety_disorders.meta.yml +++ b/etl/steps/data/grapher/fasttrack/latest/whm_treatment_gap_anxiety_disorders.meta.yml @@ -1,107 +1,64 @@ dataset: - namespace: fasttrack - version: latest - short_name: whm_treatment_gap_anxiety_disorders title: Treatment gap for anxiety disorders (WMH, 2017) - description: 'This dataset comes from the World Mental Health surveys, which conducted - national studies in 21 countries, using validated structured interviews to survey - members of the general population about symptoms of mental illnesses they had - in the past 12 months and their lifetime so far. The source describes the dataset: - "Data came from 24 community epidemiological surveys administered in 21 countries - as part of the WMH surveys (Kessler & Ustun, 2004). These included 12 surveys - carried out in high-income countries, 6 surveys in upper-middle-income countries - and 6 in low or lower-middle income countries (see table 1). The majority of surveys - were based on nationally representative household samples. Three were representative - of urban areas in their countries (Colombia, Mexico, and Peru). Three were representative - of selected regions in their countries (Japan, Nigeria, and Murcia, Spain). Four - were representative of selected Metropolitan Areas (Sao Paulo, Brazil; Medellin, - Colombia; and Beijing-Shanghai and Shenzhen in the People’s Republic of China - (PRC)). Trained lay interviewers conducted face-to-face interviews with respondents, - aged 18 years and over. The interviews took place within the households of the - respondents. To reduce respondent burden, the interview was divided into two parts. - Part I assessed core mental disorders and was administered to all respondents. - Part II, which assessed additional disorders and correlates, was administered - to all Part I respondents who met lifetime criteria for any disorder plus a probability - subsample of other Part I respondents. Part II data, the focus of this report, - were weighted by the inverse of their probabilities of selection into Part II - and additionally weighted to adjust samples to match population distributions - on the cross-classification of key socio-demographic and geographic variables. - Further details about WMH sampling and weighting are available elsewhere(Heeringa - et al., 2008). Response rates ranged between 45.9% and 97.2% and had a weighted - average of 70.1% across all surveys."' - sources: - - name: Treatment gap for anxiety disorders - World Mental Health Surveys - Alonso - et al. 2017 - published_by: Alonso et al. (2017) - description: Data comes from Community surveys of the general population - publication_year: 2017 - date_accessed: 2023-05-11 - url: https://pubmed.ncbi.nlm.nih.gov/29356216/ + description: |- + This dataset comes from the World Mental Health surveys, which conducted national studies in 21 countries, using validated structured interviews to survey members of the general population about symptoms of mental illnesses they had in the past 12 months and their lifetime so far. The source describes the dataset: "Data came from 24 community epidemiological surveys administered in 21 countries as part of the WMH surveys (Kessler & Ustun, 2004). These included 12 surveys carried out in high-income countries, 6 surveys in upper-middle-income countries and 6 in low or lower-middle income countries (see table 1). The majority of surveys were based on nationally representative household samples. Three were representative of urban areas in their countries (Colombia, Mexico, and Peru). Three were representative of selected regions in their countries (Japan, Nigeria, and Murcia, Spain). Four were representative of selected Metropolitan Areas (Sao Paulo, Brazil; Medellin, Colombia; and Beijing-Shanghai and Shenzhen in the People’s Republic of China (PRC)). Trained lay interviewers conducted face-to-face interviews with respondents, aged 18 years and over. The interviews took place within the households of the respondents. To reduce respondent burden, the interview was divided into two parts. Part I assessed core mental disorders and was administered to all respondents. Part II, which assessed additional disorders and correlates, was administered to all Part I respondents who met lifetime criteria for any disorder plus a probability subsample of other Part I respondents. Part II data, the focus of this report, were weighted by the inverse of their probabilities of selection into Part II and additionally weighted to adjust samples to match population distributions on the cross-classification of key socio-demographic and geographic variables. Further details about WMH sampling and weighting are available elsewhere(Heeringa et al., 2008). Response rates ranged between 45.9% and 97.2% and had a weighted average of 70.1% across all surveys." + licenses: + - name: Made freely available by authors + url: https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6008788/ tables: whm_treatment_gap_anxiety_disorders: variables: _12_month_anxiety_disorder: title: 12 month anxiety disorder - short_unit: '%' unit: '%' - description: Share who met the criteria for having an anxiety disorder in - the past 12 months. The Composite International Diagnostic Interview (CIDI) - was used to interview participants about symptoms of anxiety. + short_unit: '%' + description: |- + Share who met the criteria for having an anxiety disorder in the past 12 months. The Composite International Diagnostic Interview (CIDI) was used to interview participants about symptoms of anxiety. any_treatment: title: Any treatment - short_unit: '%' unit: '%' - description: Share who met the criteria for having an anxiety disorder in - the past 12 months and received any treatment for it. + short_unit: '%' + description: Share who met the criteria for having an anxiety disorder in the past 12 months and received any treatment + for it. any_treatment__conditional: title: Any treatment, conditional - short_unit: '%' unit: '%' - description: Share who received any treatment, among those who met the criteria - for an anxiety disorder in the past 12 months. + short_unit: '%' + description: Share who received any treatment, among those who met the criteria for an anxiety disorder in the past + 12 months. possibly_adequate_treatment: title: Potentially adequate treatment - short_unit: '%' unit: '%' - description: Share who met the criteria for having an anxiety disorder in - the past 12 months and received potentially adequate treatment for it. "Potentially - adequate treatment" was defined as pharmacological medication, psychotherapy, - or complementary alternative medicine. + short_unit: '%' + description: |- + Share who met the criteria for having an anxiety disorder in the past 12 months and received potentially adequate treatment for it. "Potentially adequate treatment" was defined as pharmacological medication, psychotherapy, or complementary alternative medicine. possibly_adequate_treatment__conditional: title: Potentially adequate treatment, conditional - short_unit: '%' unit: '%' - description: Share who received potentially adequate treatment, among those - who met the criteria for an anxiety disorder in the past 12 months. "Potentially - adequate treatment" was defined as pharmacological medication, psychotherapy, - or complementary alternative medicine. + short_unit: '%' + description: |- + Share who received potentially adequate treatment, among those who met the criteria for an anxiety disorder in the past 12 months. "Potentially adequate treatment" was defined as pharmacological medication, psychotherapy, or complementary alternative medicine. other_treatments: title: Other treatments - short_unit: '%' unit: '%' - description: Share who met the criteria for having an anxiety disorder in - the past 12 months and received a treatment that potentially is not adequate. - "Other treatment" was defined as seeing a religious or spiritual advisor, - social worker, or counsellor in any setting other than specialist mental - health. + short_unit: '%' + description: |- + Share who met the criteria for having an anxiety disorder in the past 12 months and received a treatment that potentially is not adequate. "Other treatment" was defined as seeing a religious or spiritual advisor, social worker, or counsellor in any setting other than specialist mental health. other_treatments__conditional: title: Other treatments, conditional - short_unit: '%' unit: '%' - description: Share who received a treatment that potentially is not adequate, - among those who met the criteria for an anxiety disorder in the past 12 - months. "Other treatment" was defined as seeing a religious or spiritual - advisor, social worker, or counsellor in any setting other than specialist - mental health. + short_unit: '%' + description: |- + Share who received a treatment that potentially is not adequate, among those who met the criteria for an anxiety disorder in the past 12 months. "Other treatment" was defined as seeing a religious or spiritual advisor, social worker, or counsellor in any setting other than specialist mental health. untreated: title: Untreated - short_unit: '%' unit: '%' - description: Share who met the criteria for having an anxiety disorder in - the past 12 months and did not receive any treatment for it. + short_unit: '%' + description: |- + Share who met the criteria for having an anxiety disorder in the past 12 months and did not receive any treatment for it. untreated__conditional: title: Untreated, conditional - short_unit: '%' unit: '%' - description: Share who did not receive any treatment, among those who met - the criteria for an anxiety disorder in the past 12 months. + short_unit: '%' + description: Share who did not receive any treatment, among those who met the criteria for an anxiety disorder in + the past 12 months. diff --git a/etl/steps/data/grapher/fasttrack/latest/whm_treatment_gap_anxiety_disorders.py b/etl/steps/data/grapher/fasttrack/latest/whm_treatment_gap_anxiety_disorders.py index b845b23302a..b05e7621518 100644 --- a/etl/steps/data/grapher/fasttrack/latest/whm_treatment_gap_anxiety_disorders.py +++ b/etl/steps/data/grapher/fasttrack/latest/whm_treatment_gap_anxiety_disorders.py @@ -1,19 +1,40 @@ import pandas as pd -from owid import catalog -from etl.helpers import PathFinder, create_dataset +from etl.helpers import PathFinder, create_dataset, get_metadata_path from etl.snapshot import Snapshot -P = PathFinder(__file__) +paths = PathFinder(__file__) def run(dest_dir: str) -> None: # load snapshot - data = pd.read_csv(Snapshot("fasttrack/latest/whm_treatment_gap_anxiety_disorders.csv").path) + snap = Snapshot("fasttrack/latest/whm_treatment_gap_anxiety_disorders.csv") - # create empty dataframe and table - tb = catalog.Table(data, short_name=P.short_name) + # load data + tb = snap.read_csv() + + # add dimensions with dim_ prefix + dims = [c for c in tb.columns if c.startswith("dim_")] + dims_without_prefix = [c[4:] for c in dims] + + if dims: + tb = tb.rename(columns={d: dw for d, dw in zip(dims, dims_without_prefix)}) + + if uses_dates(tb["year"]): + tb = tb.rename(columns={"year": "date"}).format(["country", "date"] + dims_without_prefix) + else: + tb = tb.format(["country", "year"] + dims_without_prefix) # add table, update metadata from *.meta.yml and save - ds = create_dataset(dest_dir, tables=[tb]) + ds = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # override metadata if necessary + meta_path = get_metadata_path(dest_dir).with_suffix(".override.yml") + if meta_path.exists(): + ds.update_metadata(meta_path) + ds.save() + + +def uses_dates(s: pd.Series) -> bool: + return pd.to_datetime(s, errors="coerce", format="%Y-%m-%d").notnull().all() diff --git a/etl/steps/data/grapher/growth/2022-12-19/gdp_historical.py b/etl/steps/data/grapher/growth/2022-12-19/gdp_historical.py index 2027e630167..9b8113a6433 100644 --- a/etl/steps/data/grapher/growth/2022-12-19/gdp_historical.py +++ b/etl/steps/data/grapher/growth/2022-12-19/gdp_historical.py @@ -1,6 +1,7 @@ """ Just upload GDP and GDP per capita estimations the way they are from Garden """ + from owid import catalog from etl.helpers import PathFinder diff --git a/etl/steps/data/grapher/hmd/2024-12-01/hmd.py b/etl/steps/data/grapher/hmd/2024-12-01/hmd.py new file mode 100644 index 00000000000..028b623f60c --- /dev/null +++ b/etl/steps/data/grapher/hmd/2024-12-01/hmd.py @@ -0,0 +1,83 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +INDICATORS_RELEVANT_LT = [ + "central_death_rate", + "life_expectancy", + "probability_of_death", +] + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("hmd") + + # Read table from garden dataset. + tb_lt = ds_garden.read("life_tables") + tb_exposure = ds_garden.read("exposures") + tb_deaths = ds_garden.read("deaths") + tb_pop = ds_garden.read("population") + tb_births = ds_garden.read("births") + tb_ratios = ds_garden.read("diff_ratios") + + # Filter relevant dimensions + tb_lt = keep_only_relevant_dimensions(tb_lt) + tb_exposure = keep_only_relevant_dimensions(tb_exposure) + tb_deaths = keep_only_relevant_dimensions(tb_deaths) + tb_pop = keep_only_relevant_dimensions(tb_pop) + tb_ratios = keep_only_relevant_dimensions(tb_ratios) + + # + # Save outputs. + # + cols_index = ["country", "year", "sex", "age", "type"] + tables = [ + tb_lt.format(cols_index), + tb_exposure.format(cols_index), + tb_deaths.format(["country", "year", "sex", "age"]), + tb_pop.format(["country", "year", "sex", "age"]), + tb_births.format(["country", "year", "sex"]), + tb_ratios.format(["country", "year", "age", "type"]), + ] + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=tables, check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() + + +def keep_only_relevant_dimensions(tb): + """Keep only relevant dimensions. + + - We only preserve 5-year age groups, and specific 1-year age groups. + - We only preserve 1-year observation periods. + + """ + AGES_SINGLE = [ + 0, + 10, + 15, + 25, + 45, + 65, + 80, + "total", + ] + AGES_SINGLE = list(map(str, AGES_SINGLE)) + ["110+"] + flag_1 = tb["age"].isin(AGES_SINGLE) + flag_2 = tb["age"].str.contains( + "-", + ) + + tb = tb.loc[flag_1 | flag_2] + + return tb diff --git a/etl/steps/data/grapher/hmd/2024-12-03/hmd_country.py b/etl/steps/data/grapher/hmd/2024-12-03/hmd_country.py new file mode 100644 index 00000000000..8b3a73fafe6 --- /dev/null +++ b/etl/steps/data/grapher/hmd/2024-12-03/hmd_country.py @@ -0,0 +1,28 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("hmd_country") + + # Read table from garden dataset. + tables = list(ds_garden) + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=tables, check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/homicide/2024-10-30/unodc.py b/etl/steps/data/grapher/homicide/2024-10-30/unodc.py index 08652d4cd7e..d293baadde2 100644 --- a/etl/steps/data/grapher/homicide/2024-10-30/unodc.py +++ b/etl/steps/data/grapher/homicide/2024-10-30/unodc.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/iea/2024-07-04/critical_minerals_demand_by_scenario.py b/etl/steps/data/grapher/iea/2024-07-04/critical_minerals_demand_by_scenario.py index c2fb2b50686..597924d3f9c 100644 --- a/etl/steps/data/grapher/iea/2024-07-04/critical_minerals_demand_by_scenario.py +++ b/etl/steps/data/grapher/iea/2024-07-04/critical_minerals_demand_by_scenario.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/iea/2024-07-04/critical_minerals_demand_by_technology.py b/etl/steps/data/grapher/iea/2024-07-04/critical_minerals_demand_by_technology.py index 81700c19541..a870ae2be6c 100644 --- a/etl/steps/data/grapher/iea/2024-07-04/critical_minerals_demand_by_technology.py +++ b/etl/steps/data/grapher/iea/2024-07-04/critical_minerals_demand_by_technology.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/iea/2024-07-04/critical_minerals_supply_by_country.py b/etl/steps/data/grapher/iea/2024-07-04/critical_minerals_supply_by_country.py index 3d86611e80b..3cee4db26cd 100644 --- a/etl/steps/data/grapher/iea/2024-07-04/critical_minerals_supply_by_country.py +++ b/etl/steps/data/grapher/iea/2024-07-04/critical_minerals_supply_by_country.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/iea/2024-11-20/fossil_fuel_subsidies.py b/etl/steps/data/grapher/iea/2024-11-20/fossil_fuel_subsidies.py new file mode 100644 index 00000000000..ea4798a2dbd --- /dev/null +++ b/etl/steps/data/grapher/iea/2024-11-20/fossil_fuel_subsidies.py @@ -0,0 +1,26 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("fossil_fuel_subsidies") + + # Read table from garden dataset. + tb = ds_garden.read("fossil_fuel_subsidies", reset_index=False) + + # + # Save outputs. + # + # Create a new grapher dataset. + ds_grapher = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/migration/2024-11-19/migration_distance.py b/etl/steps/data/grapher/migration/2024-11-19/migration_distance.py new file mode 100644 index 00000000000..1529dcdeaef --- /dev/null +++ b/etl/steps/data/grapher/migration/2024-11-19/migration_distance.py @@ -0,0 +1,31 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("migration_distance") + + # Read table from garden dataset. + tb = ds_garden["migration_distance"].reset_index() + + tb = tb.rename(columns={"country_origin": "country"}) + + tb = tb.format(["country", "year"]) + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/migration/2024-11-20/migrant_stock_flows.meta.yml b/etl/steps/data/grapher/migration/2024-11-20/migrant_stock_flows.meta.yml new file mode 100644 index 00000000000..3e7fc3f6f09 --- /dev/null +++ b/etl/steps/data/grapher/migration/2024-11-20/migrant_stock_flows.meta.yml @@ -0,0 +1,16 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Migration + + +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +dataset: + update_period_days: 365 + title: International Migrant Stock (Origin and Destination) + + + diff --git a/etl/steps/data/grapher/migration/2024-11-20/migrant_stock_flows.py b/etl/steps/data/grapher/migration/2024-11-20/migrant_stock_flows.py new file mode 100644 index 00000000000..71ad1d1821e --- /dev/null +++ b/etl/steps/data/grapher/migration/2024-11-20/migrant_stock_flows.py @@ -0,0 +1,82 @@ +"""Load a garden dataset and create a grapher dataset. +This grapher step has two purposes: +1. Format the data in a way that is compatible with the grapher database (split into two tables and index on country and year). +2. Add metadata programmatically to the data.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("migrant_stock") + + # Read table from garden dataset. + tb = ds_garden.read("migrant_stock_dest_origin") + + tb = tb.drop(columns=["migrants_female", "migrants_male"]) + + tb_dest_cols = tb.pivot( + index=["country_origin", "year"], + columns="country_destination", + values=["migrants_all_sexes"], + ) + + tb_dest_cols.columns = [col[0] + "_to_" + col[1] for col in tb_dest_cols.columns] + + tb_origin_cols = tb.pivot( + index=["country_destination", "year"], + columns="country_origin", + values=["migrants_all_sexes"], + ) + + tb_origin_cols.columns = [col[0] + "_from_" + col[1] for col in tb_origin_cols.columns] + + # add metadata: + + for col in tb_dest_cols.columns: + dest = col.split("migrants_all_sexes_to_")[1] + tb_dest_cols[col].metadata.unit = "people" + tb_dest_cols[col].metadata.short_unit = "" + tb_dest_cols[col].metadata.title = f"Number of immigrants who moved to {dest}" + tb_dest_cols[ + col + ].metadata.description_short = f"Number of migrants who have moved to {dest}. The numbers describe cumulative migrant stock, not migrants who moved in this year." + + for col in tb_origin_cols.columns: + origin = col.split("migrants_all_sexes_from_")[1] + + tb_origin_cols[col].metadata.unit = "people" + tb_origin_cols[col].metadata.short_unit = "" + tb_origin_cols[col].metadata.title = f"Number of emigrants who moved from {origin}" + tb_origin_cols[ + col + ].metadata.description_short = f"Number of migrants who have moved to away from {origin}. The numbers describe cumulative migrant stock, not migrants who moved in this year." + + tb_dest_cols = tb_dest_cols.reset_index() + tb_dest_cols = tb_dest_cols.rename(columns={"country_origin": "country"}) + tb_dest_cols.metadata.short_name = "migrant_stock_origin" + tb_dest_cols = tb_dest_cols.format(["country", "year"]) + + tb_origin_cols = tb_origin_cols.reset_index() + tb_origin_cols = tb_origin_cols.rename(columns={"country_destination": "country"}) + tb_origin_cols.metadata.short_name = "migrant_stock_destination" + tb_origin_cols = tb_origin_cols.format(["country", "year"]) + + # Save outputs + # + # Create a new grapher dataset with the same metadata as the garden dataset + ds_grapher = create_dataset( + dest_dir, + tables=[tb_origin_cols, tb_dest_cols], + check_variables_metadata=True, + default_metadata=ds_garden.metadata, + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/minerals/2024-07-15/global_mine_production_by_mineral.py b/etl/steps/data/grapher/minerals/2024-07-15/global_mine_production_by_mineral.py new file mode 100644 index 00000000000..c048d005bd4 --- /dev/null +++ b/etl/steps/data/grapher/minerals/2024-07-15/global_mine_production_by_mineral.py @@ -0,0 +1,153 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +# Columns to select and how to rename them. +COLUMNS = { + "country": "country", + "year": "year", + "production_antimony_mine_tonnes": "Antimony", + "production_asbestos_mine_tonnes": "Asbestos", + "production_barite_mine_tonnes": "Barite", + "production_bauxite_mine_tonnes": "Bauxite", + "production_beryllium_mine_tonnes": "Beryllium", + "production_bismuth_mine_tonnes": "Bismuth", + "production_boron_mine_tonnes": "Boron", + "production_chromium_mine_tonnes": "Chromium", + "production_coal_mine_tonnes": "Coal", + "production_cobalt_mine_tonnes": "Cobalt", + "production_coltan_mine__columbite_tantalite_tonnes": "Coltan (columbite-tantalite)", + "production_coltan_mine__columbite_tonnes": "Coltan (columbite)", + "production_coltan_mine__tantalite_tonnes": "Coltan (tantalite)", + "production_copper_mine_tonnes": "Copper", + "production_diamond_mine__industrial_tonnes": "Diamond (industrial)", + "production_diamond_mine_and_synthetic__industrial_tonnes": "Diamond and synthetic (industrial)", + "production_feldspar_mine_tonnes": "Feldspar", + "production_fluorspar_mine_tonnes": "Fluorspar", + "production_garnet_mine_tonnes": "Garnet", + "production_gemstones_mine_tonnes": "Gemstones", + "production_gold_mine_tonnes": "Gold", + "production_graphite_mine_tonnes": "Graphite", + "production_gypsum_mine_tonnes": "Gypsum", + "production_helium_mine_tonnes": "Helium", + "production_iodine_mine_tonnes": "Iodine", + "production_iron_ore_mine__crude_ore_tonnes": "Iron ore (crude ore)", + "production_iron_ore_mine__iron_content_tonnes": "Iron ore (iron content)", + "production_lead_mine_tonnes": "Lead", + "production_lithium_mine_tonnes": "Lithium", + "production_magnesium_compounds_mine_tonnes": "Magnesium compounds", + "production_manganese_mine_tonnes": "Manganese", + "production_mercury_mine_tonnes": "Mercury", + "production_mica_mine__scrap_and_flake_tonnes": "Mica (scrap and flake)", + "production_mica_mine__sheet_tonnes": "Mica (sheet)", + "production_mica_mine_tonnes": "Mica", + "production_molybdenum_mine_tonnes": "Molybdenum", + "production_nickel_mine_tonnes": "Nickel", + "production_niobium_mine__pyrochlore_tonnes": "Niobium (pyrochlore)", + "production_niobium_mine_tonnes": "Niobium", + "production_phosphate_rock_mine__aluminum_phosphate_tonnes": "Phosphate rock (aluminum phosphate)", + "production_phosphate_rock_mine_tonnes": "Phosphate rock", + "production_platinum_group_metals_mine__iridium_tonnes": "Platinum group metals (iridium)", + "production_platinum_group_metals_mine__other_tonnes": "Platinum group metals (other)", + "production_platinum_group_metals_mine__palladium_tonnes": "Platinum group metals (palladium)", + "production_platinum_group_metals_mine__platinum_tonnes": "Platinum group metals (platinum)", + "production_platinum_group_metals_mine__rhodium_tonnes": "Platinum group metals (rhodium)", + "production_platinum_group_metals_mine__ruthenium_tonnes": "Platinum group metals (ruthenium)", + "production_potash_mine__chloride_tonnes": "Potash (chloride)", + "production_potash_mine__polyhalite_tonnes": "Potash (polyhalite)", + "production_potash_mine__potassic_salts_tonnes": "Potash (potassic salts)", + "production_potash_mine_tonnes": "Potash", + "production_rare_earths_mine_tonnes": "Rare earths", + "production_salt_mine_tonnes": "Salt", + "production_sand_and_gravel_mine__construction_tonnes": "Sand and gravel (construction)", + "production_sand_and_gravel_mine__industrial_tonnes": "Sand and gravel (industrial)", + "production_silver_mine_tonnes": "Silver", + "production_strontium_mine_tonnes": "Strontium", + "production_talc_and_pyrophyllite_mine__pyrophyllite_tonnes": "Talc and pyrophyllite (pyrophyllite)", + "production_talc_and_pyrophyllite_mine_tonnes": "Talc and pyrophyllite", + "production_tantalum_mine_tonnes": "Tantalum", + "production_tin_mine_tonnes": "Tin", + "production_titanium_mine__ilmenite_tonnes": "Titanium (ilmenite)", + "production_titanium_mine__rutile_tonnes": "Titanium (rutile)", + "production_tungsten_mine_tonnes": "Tungsten", + "production_uranium_mine_tonnes": "Uranium", + "production_vanadium_mine_tonnes": "Vanadium", + "production_zinc_mine_tonnes": "Zinc", + "production_zirconium_and_hafnium_mine_tonnes": "Zirconium and hafnium", +} + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset and read its flat table. + ds_garden = paths.load_dataset("minerals") + tb = ds_garden.read("minerals") + + # + # Process data. + # + # Select and rename columns. + tb = tb[COLUMNS.keys()].rename(columns=COLUMNS, errors="raise") + + # Select global data. + tb = tb[tb["country"] == "World"].drop(columns=["country"], errors="raise").reset_index(drop=True) + + # Gather all descriptions. + # NOTE: Footnotes will be gathered and used as description key. + # description_key = [] + _description_processing = [] + # _description_from_producer = [] + footnotes = [] + for column in tb.drop(columns=["year"]).columns: + grapher_config = tb[column].m.presentation.grapher_config + if grapher_config: + # NOTE: Ignore the frequent footnote saying that "The sum of all countries may exceed World data on certain years (by up to 10%), due to discrepancies between data sources." + if "The sum of all countries" not in grapher_config["note"]: + footnotes.append(f"{column} - {grapher_config['note']}") + # description_key.append(tb[column].metadata.description_short) + _description_processing.append(tb[column].metadata.description_processing) + # _description_from_producer.append(f"- {column}\n{tb[column].metadata.description_from_producer}") + _description_processing = sorted( + set([tb[column].metadata.description_processing for column in tb.drop(columns=["year"]).columns]) + ) + # By construction, processing description should be the same for all indicators. + assert len(_description_processing) == 1, "All columns were expected to have the same processing description." + description_processing = _description_processing[0] + # Gather all descriptions from producer. + # description_from_producer = "\n".join(["\n ".join(description.split("\n")) for description in _description_from_producer if description]) + + # Melt table to create a long table with mineral as "country" column. + tb_long = tb.melt(id_vars=["year"], var_name="country", value_name="mine_production") + + # Drop empty rows. + tb_long = tb_long.dropna(subset=["mine_production"]) + + # Improve metadata. + tb_long["mine_production"].metadata.title = "Global mine production of different minerals" + tb_long["mine_production"].metadata.unit = "tonnes" + tb_long["mine_production"].metadata.short_unit = "t" + tb_long[ + "mine_production" + ].metadata.description_short = ( + "Measured in tonnes of mined, rather than [refined](#dod:refined-production) production." + ) + tb_long["mine_production"].metadata.description_key = footnotes + tb_long["mine_production"].metadata.description_processing = description_processing + # NOTE: The following metadata is too long and cannot be inserted in DB. + # tb_long["mine_production"].metadata.description_from_producer = description_from_producer + tb_long.metadata.title = "Global mine production by mineral" + + # Improve table format. + tb_long = tb_long.format(short_name=paths.short_name) + + # + # Save outputs. + # + # Create a new grapher dataset. + ds_grapher = create_dataset(dest_dir, tables=[tb_long], check_variables_metadata=True) + ds_grapher.save() diff --git a/etl/steps/data/grapher/oecd/2023-09-21/plastic_emissions.py b/etl/steps/data/grapher/oecd/2023-09-21/plastic_emissions.py index 37002450a15..5307d4fa6ec 100644 --- a/etl/steps/data/grapher/oecd/2023-09-21/plastic_emissions.py +++ b/etl/steps/data/grapher/oecd/2023-09-21/plastic_emissions.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/oecd/2023-09-21/plastic_use_application.py b/etl/steps/data/grapher/oecd/2023-09-21/plastic_use_application.py index af25fc5338c..db89e2ba131 100644 --- a/etl/steps/data/grapher/oecd/2023-09-21/plastic_use_application.py +++ b/etl/steps/data/grapher/oecd/2023-09-21/plastic_use_application.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/oecd/2023-09-21/plastic_use_polymer.py b/etl/steps/data/grapher/oecd/2023-09-21/plastic_use_polymer.py index e76366d442a..fd509f4f843 100644 --- a/etl/steps/data/grapher/oecd/2023-09-21/plastic_use_polymer.py +++ b/etl/steps/data/grapher/oecd/2023-09-21/plastic_use_polymer.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/owid/latest/key_indicators.py b/etl/steps/data/grapher/owid/latest/key_indicators.py index 17311c164b9..e3b7e2b3d95 100644 --- a/etl/steps/data/grapher/owid/latest/key_indicators.py +++ b/etl/steps/data/grapher/owid/latest/key_indicators.py @@ -96,9 +96,9 @@ def _add_metric_new( if title_suffix: table[metric_new].metadata.title = f"{table[metric_new].metadata.title} {title_suffix}" if display_name_suffix: - table[metric_new].metadata.display[ - "name" - ] = f"{table[metric_new].metadata.display['name']} {display_name_suffix}" + table[metric_new].metadata.display["name"] = ( + f"{table[metric_new].metadata.display['name']} {display_name_suffix}" + ) table[metric_new].metadata.description = description # Get dtype diff --git a/etl/steps/data/grapher/papers/2023-07-10/farmer_lafond_2016.py b/etl/steps/data/grapher/papers/2023-07-10/farmer_lafond_2016.py index 78b5ce5932c..dca8de0a979 100644 --- a/etl/steps/data/grapher/papers/2023-07-10/farmer_lafond_2016.py +++ b/etl/steps/data/grapher/papers/2023-07-10/farmer_lafond_2016.py @@ -1,6 +1,4 @@ -"""Load garden dataset for Farmer & Lafond (2016) data and create a grapher dataset. - -""" +"""Load garden dataset for Farmer & Lafond (2016) data and create a grapher dataset.""" from owid.catalog import Dataset diff --git a/etl/steps/data/grapher/papers/2023-12-12/farmer_lafond_2016.py b/etl/steps/data/grapher/papers/2023-12-12/farmer_lafond_2016.py index 29a7aa0a7ea..cf4baa28346 100644 --- a/etl/steps/data/grapher/papers/2023-12-12/farmer_lafond_2016.py +++ b/etl/steps/data/grapher/papers/2023-12-12/farmer_lafond_2016.py @@ -1,6 +1,4 @@ -"""Load garden dataset for Farmer & Lafond (2016) data and create a grapher dataset. - -""" +"""Load garden dataset for Farmer & Lafond (2016) data and create a grapher dataset.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/grapher/research_development/2024-05-20/patents_articles.py b/etl/steps/data/grapher/research_development/2024-05-20/patents_articles.py index 690edf82e71..07796c66b19 100644 --- a/etl/steps/data/grapher/research_development/2024-05-20/patents_articles.py +++ b/etl/steps/data/grapher/research_development/2024-05-20/patents_articles.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/rff/2023-10-19/emissions_weighted_carbon_price.py b/etl/steps/data/grapher/rff/2023-10-19/emissions_weighted_carbon_price.py index 1bda1435cd3..0f7ff18d03b 100644 --- a/etl/steps/data/grapher/rff/2023-10-19/emissions_weighted_carbon_price.py +++ b/etl/steps/data/grapher/rff/2023-10-19/emissions_weighted_carbon_price.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/rff/2023-10-19/world_carbon_pricing.py b/etl/steps/data/grapher/rff/2023-10-19/world_carbon_pricing.py index 54a1a73694e..237f4ed023f 100644 --- a/etl/steps/data/grapher/rff/2023-10-19/world_carbon_pricing.py +++ b/etl/steps/data/grapher/rff/2023-10-19/world_carbon_pricing.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/survey/2023-08-04/trust_surveys.py b/etl/steps/data/grapher/survey/2023-08-04/trust_surveys.py index 0a56e5074d1..1429c30baad 100644 --- a/etl/steps/data/grapher/survey/2023-08-04/trust_surveys.py +++ b/etl/steps/data/grapher/survey/2023-08-04/trust_surveys.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from etl.helpers import PathFinder, create_dataset, grapher_checks # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/un/2023-01-24/un_sdg.py b/etl/steps/data/grapher/un/2023-01-24/un_sdg.py index 09c0febbf50..3db820ca9e8 100644 --- a/etl/steps/data/grapher/un/2023-01-24/un_sdg.py +++ b/etl/steps/data/grapher/un/2023-01-24/un_sdg.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + import json import os from functools import cache diff --git a/etl/steps/data/grapher/un/2023-08-16/un_sdg.py b/etl/steps/data/grapher/un/2023-08-16/un_sdg.py index 4b36809be1b..c45723b779e 100644 --- a/etl/steps/data/grapher/un/2023-08-16/un_sdg.py +++ b/etl/steps/data/grapher/un/2023-08-16/un_sdg.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + import json import os import re diff --git a/etl/steps/data/grapher/un/2024-01-17/urban_agglomerations_largest_cities.py b/etl/steps/data/grapher/un/2024-01-17/urban_agglomerations_largest_cities.py index e922b1b7ebe..10e396c4de9 100644 --- a/etl/steps/data/grapher/un/2024-01-17/urban_agglomerations_largest_cities.py +++ b/etl/steps/data/grapher/un/2024-01-17/urban_agglomerations_largest_cities.py @@ -31,15 +31,15 @@ def run(dest_dir: str) -> None: ) tb = tb.drop(columns=["rank_order", "population_capital", "country_code"]) - # Create two new dataframes to separate data into estimates and projections (pre-2019 and post-2019) - past_estimates = tb[tb["year"] < 2019].copy() - future_projections = tb[tb["year"] >= 2019].copy() + # Create two new dataframes to separate data into estimates and projections + past_estimates = tb[tb["year"] <= 2015].copy() + future_projections = tb[tb["year"] >= 2015].copy() # Now, for each column in the original dataframe, split it into two for col in tb.columns: if col not in ["country", "year"]: - past_estimates[f"{col}_estimates"] = tb.loc[tb["year"] < 2019, col] - future_projections[f"{col}_projections"] = tb.loc[tb["year"] >= 2019, col] + past_estimates[f"{col}_estimates"] = tb.loc[tb["year"] <= 2015, col] + future_projections[f"{col}_projections"] = tb.loc[tb["year"] >= 2015, col] past_estimates = past_estimates.drop(columns=[col]) future_projections = future_projections.drop(columns=[col]) diff --git a/etl/steps/data/grapher/un/2024-03-14/un_wpp_most.py b/etl/steps/data/grapher/un/2024-03-14/un_wpp_most.py index d3919bc2c93..dd1bd352efd 100644 --- a/etl/steps/data/grapher/un/2024-03-14/un_wpp_most.py +++ b/etl/steps/data/grapher/un/2024-03-14/un_wpp_most.py @@ -15,12 +15,12 @@ def run(dest_dir: str) -> None: # Read five-year age-group table from garden dataset. tb_five = ds_garden["population_5_year_age_groups"].reset_index() - tb_five = tb_five.rename(columns={"location": "country"}) + # tb_five = tb_five.rename(columns={"location": "country"}) tb_five = tb_five.set_index(["country", "year"], verify_integrity=True) # Read ten-year age-group table from garden dataset. tb_ten = ds_garden["population_10_year_age_groups"].reset_index() - tb_ten = tb_ten.rename(columns={"location": "country"}) + # tb_ten = tb_ten.rename(columns={"location": "country"}) tb_ten = tb_ten.set_index(["country", "year"], verify_integrity=True) # Save outputs. # diff --git a/etl/steps/data/grapher/un/2024-08-27/un_sdg.py b/etl/steps/data/grapher/un/2024-08-27/un_sdg.py index 92aa4d73b94..dec68286c3d 100644 --- a/etl/steps/data/grapher/un/2024-08-27/un_sdg.py +++ b/etl/steps/data/grapher/un/2024-08-27/un_sdg.py @@ -1,4 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" + import json import os import re diff --git a/etl/steps/data/grapher/unep/2023-01-03/renewable_energy_investments.py b/etl/steps/data/grapher/unep/2023-01-03/renewable_energy_investments.py index 5df255a51a1..554878cf0aa 100644 --- a/etl/steps/data/grapher/unep/2023-01-03/renewable_energy_investments.py +++ b/etl/steps/data/grapher/unep/2023-01-03/renewable_energy_investments.py @@ -1,6 +1,4 @@ -"""Load renewable energy investments data from garden and create a grapher dataset. - -""" +"""Load renewable energy investments data from garden and create a grapher dataset.""" from owid import catalog diff --git a/etl/steps/data/grapher/unep/2023-12-12/renewable_energy_investments.py b/etl/steps/data/grapher/unep/2023-12-12/renewable_energy_investments.py index f9553e50be6..37a0a93c36f 100644 --- a/etl/steps/data/grapher/unep/2023-12-12/renewable_energy_investments.py +++ b/etl/steps/data/grapher/unep/2023-12-12/renewable_energy_investments.py @@ -1,6 +1,5 @@ -"""Load renewable energy investments data from garden and create a grapher dataset. +"""Load renewable energy investments data from garden and create a grapher dataset.""" -""" from etl.helpers import PathFinder, create_dataset # Load paths and naming conventions. diff --git a/etl/steps/data/grapher/urbanization/2024-12-02/ghsl_urban_centers.py b/etl/steps/data/grapher/urbanization/2024-12-02/ghsl_urban_centers.py new file mode 100644 index 00000000000..7ed2180485c --- /dev/null +++ b/etl/steps/data/grapher/urbanization/2024-12-02/ghsl_urban_centers.py @@ -0,0 +1,28 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("ghsl_urban_centers") + + # Read table from garden dataset. + tb = ds_garden.read("ghsl_urban_centers", reset_index=False) + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/wb/2024-12-03/poverty_projections.py b/etl/steps/data/grapher/wb/2024-12-03/poverty_projections.py new file mode 100644 index 00000000000..08e7178862b --- /dev/null +++ b/etl/steps/data/grapher/wb/2024-12-03/poverty_projections.py @@ -0,0 +1,28 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("poverty_projections") + + # Read table from garden dataset. + tb = ds_garden.read("poverty_projections", reset_index=False) + + # + # Save outputs. + # + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_garden.metadata + ) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/grapher/who/2024-02-14/gho_suicides.py b/etl/steps/data/grapher/who/2024-02-14/gho_suicides.py index 43ce1fa023b..9ef92f7890d 100644 --- a/etl/steps/data/grapher/who/2024-02-14/gho_suicides.py +++ b/etl/steps/data/grapher/who/2024-02-14/gho_suicides.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/who/2024-05-20/vehicles.py b/etl/steps/data/grapher/who/2024-05-20/vehicles.py index 5ca3c9ffa49..09127ac872f 100644 --- a/etl/steps/data/grapher/who/2024-05-20/vehicles.py +++ b/etl/steps/data/grapher/who/2024-05-20/vehicles.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/who/2024-07-26/mortality_database.py b/etl/steps/data/grapher/who/2024-07-26/mortality_database.py index c6d590c1360..f13a39b66f3 100644 --- a/etl/steps/data/grapher/who/2024-07-26/mortality_database.py +++ b/etl/steps/data/grapher/who/2024-07-26/mortality_database.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from etl.helpers import PathFinder, create_dataset, grapher_checks # Get paths and naming conventions for current step. diff --git a/etl/steps/data/grapher/who/2024-07-30/ghe.py b/etl/steps/data/grapher/who/2024-07-30/ghe.py index 1c5b6eaa58c..a4fba2838fc 100644 --- a/etl/steps/data/grapher/who/2024-07-30/ghe.py +++ b/etl/steps/data/grapher/who/2024-07-30/ghe.py @@ -20,8 +20,8 @@ def run(dest_dir: str) -> None: # Save outputs. # tables = [ - tb_garden, tb_garden_ratio, + tb_garden, ] # Create a new grapher dataset with the same metadata as the garden dataset. diff --git a/etl/steps/data/grapher/wpf/2024-10-03/famines.py b/etl/steps/data/grapher/wpf/2024-10-03/famines.py index 21034f3119e..099f31da298 100644 --- a/etl/steps/data/grapher/wpf/2024-10-03/famines.py +++ b/etl/steps/data/grapher/wpf/2024-10-03/famines.py @@ -1,6 +1,5 @@ """Load a garden dataset and create a grapher dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/agriculture/2024-05-23/fao_1949.py b/etl/steps/data/meadow/agriculture/2024-05-23/fao_1949.py index 4ad3adcb1ed..f20db4edcc8 100644 --- a/etl/steps/data/meadow/agriculture/2024-05-23/fao_1949.py +++ b/etl/steps/data/meadow/agriculture/2024-05-23/fao_1949.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/agriculture/2024-05-23/fao_2000.py b/etl/steps/data/meadow/agriculture/2024-05-23/fao_2000.py index 57380e279b2..4755cdea093 100644 --- a/etl/steps/data/meadow/agriculture/2024-05-23/fao_2000.py +++ b/etl/steps/data/meadow/agriculture/2024-05-23/fao_2000.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/agriculture/2024-05-23/fogel_2004.py b/etl/steps/data/meadow/agriculture/2024-05-23/fogel_2004.py index 37003fd7d70..ab1066b289d 100644 --- a/etl/steps/data/meadow/agriculture/2024-05-23/fogel_2004.py +++ b/etl/steps/data/meadow/agriculture/2024-05-23/fogel_2004.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/agriculture/2024-05-23/grigg_1995.py b/etl/steps/data/meadow/agriculture/2024-05-23/grigg_1995.py index 82caf2002e5..9159cf0bbe5 100644 --- a/etl/steps/data/meadow/agriculture/2024-05-23/grigg_1995.py +++ b/etl/steps/data/meadow/agriculture/2024-05-23/grigg_1995.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/agriculture/2024-05-23/jonsson_1998.py b/etl/steps/data/meadow/agriculture/2024-05-23/jonsson_1998.py index f7b97a260a4..0925d95e407 100644 --- a/etl/steps/data/meadow/agriculture/2024-05-23/jonsson_1998.py +++ b/etl/steps/data/meadow/agriculture/2024-05-23/jonsson_1998.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/animal_welfare/2023-09-01/chick_culling_laws.py b/etl/steps/data/meadow/animal_welfare/2023-09-01/chick_culling_laws.py index c0e2889058e..11d14575a07 100644 --- a/etl/steps/data/meadow/animal_welfare/2023-09-01/chick_culling_laws.py +++ b/etl/steps/data/meadow/animal_welfare/2023-09-01/chick_culling_laws.py @@ -1,6 +1,4 @@ -"""Load a snapshot and create a meadow dataset. - -""" +"""Load a snapshot and create a meadow dataset.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/meadow/animal_welfare/2023-09-05/bullfighting_laws.py b/etl/steps/data/meadow/animal_welfare/2023-09-05/bullfighting_laws.py index 0a03346d0d9..229dea290f3 100644 --- a/etl/steps/data/meadow/animal_welfare/2023-09-05/bullfighting_laws.py +++ b/etl/steps/data/meadow/animal_welfare/2023-09-05/bullfighting_laws.py @@ -1,6 +1,4 @@ -"""Load a snapshot and create a meadow dataset. - -""" +"""Load a snapshot and create a meadow dataset.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/meadow/animal_welfare/2023-10-24/fur_laws.py b/etl/steps/data/meadow/animal_welfare/2023-10-24/fur_laws.py index d6c740e6a3e..61b7052b97e 100644 --- a/etl/steps/data/meadow/animal_welfare/2023-10-24/fur_laws.py +++ b/etl/steps/data/meadow/animal_welfare/2023-10-24/fur_laws.py @@ -1,6 +1,4 @@ -"""Load a snapshot and create a meadow dataset. - -""" +"""Load a snapshot and create a meadow dataset.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/meadow/animal_welfare/2024-06-04/bullfighting_laws.py b/etl/steps/data/meadow/animal_welfare/2024-06-04/bullfighting_laws.py index 0a03346d0d9..229dea290f3 100644 --- a/etl/steps/data/meadow/animal_welfare/2024-06-04/bullfighting_laws.py +++ b/etl/steps/data/meadow/animal_welfare/2024-06-04/bullfighting_laws.py @@ -1,6 +1,4 @@ -"""Load a snapshot and create a meadow dataset. - -""" +"""Load a snapshot and create a meadow dataset.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/meadow/animal_welfare/2024-09-13/fur_laws.py b/etl/steps/data/meadow/animal_welfare/2024-09-13/fur_laws.py index 931e4bd286a..4dfa99c210e 100644 --- a/etl/steps/data/meadow/animal_welfare/2024-09-13/fur_laws.py +++ b/etl/steps/data/meadow/animal_welfare/2024-09-13/fur_laws.py @@ -1,6 +1,4 @@ -"""Load a snapshot and create a meadow dataset. - -""" +"""Load a snapshot and create a meadow dataset.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/meadow/antibiotics/2024-12-02/microbe_amr.py b/etl/steps/data/meadow/antibiotics/2024-12-02/microbe_amr.py new file mode 100644 index 00000000000..6dd02b0b3a3 --- /dev/null +++ b/etl/steps/data/meadow/antibiotics/2024-12-02/microbe_amr.py @@ -0,0 +1,36 @@ +"""Load a snapshot and create a meadow dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("microbe_amr.csv") + + # Load data from snapshot. + tb = snap.read() + assert all(tb["Age"] == "Neonatal") + + # + # Process data. + # + tb = tb.drop(columns=["Age", "Sex", "Measure", "Metric", "Pathogen"]) + tb = tb.rename(columns={"Location": "country", "Year": "year"}) + + # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. + tb = tb.format(["country", "year", "infectious_syndrome", "counterfactual"]) + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=snap.metadata) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/antibiotics/2024-12-02/microbe_neonatal_amr.py b/etl/steps/data/meadow/antibiotics/2024-12-02/microbe_neonatal_amr.py new file mode 100644 index 00000000000..a1126c3a231 --- /dev/null +++ b/etl/steps/data/meadow/antibiotics/2024-12-02/microbe_neonatal_amr.py @@ -0,0 +1,40 @@ +"""Load a snapshot and create a meadow dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("microbe_neonatal_amr.csv") + + # Load data from snapshot. + tb = snap.read() + assert all(tb["Age"] == "Neonatal") + assert len(tb["Counterfactual"].unique()) == 1 + assert all(tb["Infectious syndrome"] == "Bloodstream infections") + # + # Process data. + # + tb = tb.drop(columns=["Age", "Sex", "Measure", "Metric", "Infectious syndrome", "Pathogen Type", "Counterfactual"]) + tb = tb.rename(columns={"Location": "country", "Year": "year"}) + + # + # Process data. + # + # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. + tb = tb.format(["country", "year", "pathogen"]) + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=snap.metadata) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/antibiotics/2024-12-02/total_pathogen_bloodstream.py b/etl/steps/data/meadow/antibiotics/2024-12-02/total_pathogen_bloodstream.py new file mode 100644 index 00000000000..63ef94b626b --- /dev/null +++ b/etl/steps/data/meadow/antibiotics/2024-12-02/total_pathogen_bloodstream.py @@ -0,0 +1,40 @@ +"""Load a snapshot and create a meadow dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("total_pathogen_bloodstream.csv") + + # Load data from snapshot. + tb = snap.read() + # Checking the right slice of data has been uploaded + assert all(tb["Age"] == "All Ages") + assert all(tb["Sex"] == "Both sexes") + assert all(tb["Measure"] == "Deaths") + assert all(tb["Metric"] == "Number") + assert all(tb["Counterfactual"] == "Total") + assert all(tb["Infectious syndrome"] == "Bloodstream infections") + + # + # Process data. + tb = tb.drop(columns=["Age", "Sex", "Measure", "Metric", "Infectious syndrome", "Counterfactual"]) + tb = tb.rename(columns={"Location": "country", "Year": "year"}) + # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. + tb = tb.format(["country", "year", "pathogen"]) + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=snap.metadata) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.py b/etl/steps/data/meadow/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.py new file mode 100644 index 00000000000..80b21a69747 --- /dev/null +++ b/etl/steps/data/meadow/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.py @@ -0,0 +1,42 @@ +"""Load a snapshot and create a meadow dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("total_pathogen_bloodstream_amr.csv") + + # Load data from snapshot. + tb = snap.read() + # Checking the right slice of data has been uploaded + assert all(tb["Age"] == "All Ages") + assert all(tb["Sex"] == "Both sexes") + assert all(tb["Measure"] == "Deaths") + assert all(tb["Metric"] == "Number") + assert all(tb["Counterfactual"] == "Attributable") + assert all(tb["Infectious syndrome"] == "Bloodstream infections") + + # + # Process data. + # + # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. + tb = tb.drop(columns=["Age", "Sex", "Measure", "Metric", "Infectious syndrome", "Counterfactual"]) + tb = tb.rename(columns={"Location": "country", "Year": "year"}) + # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. + tb = tb.format(["country", "year", "pathogen"]) + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=snap.metadata) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/antibiotics/2024-12-03/glass_enrolment.py b/etl/steps/data/meadow/antibiotics/2024-12-03/glass_enrolment.py new file mode 100644 index 00000000000..9c94caea506 --- /dev/null +++ b/etl/steps/data/meadow/antibiotics/2024-12-03/glass_enrolment.py @@ -0,0 +1,38 @@ +"""Load a snapshot and create a meadow dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("glass_enrolment.xlsx") + + # Load data from snapshot. + tb = snap.read() + # Drop the rows where there isn't a country name + tb = tb.dropna(subset=["Code"]) + + # Check the number of countries + assert len(tb["Country"] == 197) + # Rename columns + tb = tb.drop(columns=["Country"]).rename(columns={"Label": "country"}) + tb["year"] = snap.metadata.origin.date_published.split("-")[0] + # Process data. + # + # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. + tb = tb.format(["country", "year"]) + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=snap.metadata) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/antibiotics/2024-12-04/microbe_total_pathogens.py b/etl/steps/data/meadow/antibiotics/2024-12-04/microbe_total_pathogens.py new file mode 100644 index 00000000000..f03016ef003 --- /dev/null +++ b/etl/steps/data/meadow/antibiotics/2024-12-04/microbe_total_pathogens.py @@ -0,0 +1,39 @@ +"""Load a snapshot and create a meadow dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("microbe_total_pathogens.csv") + + # Load data from snapshot. + tb = snap.read() + assert all(tb["Age"] == "All Ages") + assert all(tb["Sex"] == "Both sexes") + assert all(tb["Measure"] == "Deaths") + assert all(tb["Metric"] == "Number") + assert all(tb["Counterfactual"] == "Total") + assert all(tb["Infectious syndrome"] == "All infectious syndromes") + + # + # Process data. + tb = tb.drop(columns=["Age", "Sex", "Measure", "Metric", "Infectious syndrome", "Counterfactual"]) + tb = tb.rename(columns={"Location": "country", "Year": "year", "Pathogen": "pathogen"}) + # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. + tb = tb.format(["country", "year", "pathogen"]) + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=snap.metadata) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/antibiotics/2024-12-04/microbe_total_pathogens_amr.py b/etl/steps/data/meadow/antibiotics/2024-12-04/microbe_total_pathogens_amr.py new file mode 100644 index 00000000000..0e88870141e --- /dev/null +++ b/etl/steps/data/meadow/antibiotics/2024-12-04/microbe_total_pathogens_amr.py @@ -0,0 +1,39 @@ +"""Load a snapshot and create a meadow dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("microbe_total_pathogens_amr.csv") + + # Load data from snapshot. + tb = snap.read() + assert all(tb["Age"] == "All Ages") + assert all(tb["Sex"] == "Both sexes") + assert all(tb["Measure"] == "Deaths") + assert all(tb["Metric"] == "Number") + assert all(tb["Counterfactual"] == "Attributable") + assert all(tb["Infectious syndrome"] == "All infectious syndromes") + + # + # Process data. + tb = tb.drop(columns=["Age", "Sex", "Measure", "Metric", "Infectious syndrome", "Counterfactual"]) + tb = tb.rename(columns={"Location": "country", "Year": "year", "Pathogen": "pathogen"}) + # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. + tb = tb.format(["country", "year", "pathogen"]) + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=snap.metadata) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/antibiotics/2024-12-05/microbe_neonatal_total_amr.py b/etl/steps/data/meadow/antibiotics/2024-12-05/microbe_neonatal_total_amr.py new file mode 100644 index 00000000000..b79dae1416b --- /dev/null +++ b/etl/steps/data/meadow/antibiotics/2024-12-05/microbe_neonatal_total_amr.py @@ -0,0 +1,39 @@ +"""Load a snapshot and create a meadow dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("microbe_neonatal_total_amr.csv") + + # Load data from snapshot. + tb = snap.read() + assert all(tb["Age"] == "Neonatal") + assert all(tb["Sex"] == "Both sexes") + assert all(tb["Measure"] == "Deaths") + assert all(tb["Metric"] == "Number") + assert all(tb["Counterfactual"] == "Attributable") + assert all(tb["Infectious syndrome"] == "All infectious syndromes") + + # + # Process data. + tb = tb.drop(columns=["Age", "Sex", "Measure", "Metric", "Infectious syndrome", "Counterfactual"]) + tb = tb.rename(columns={"Location": "country", "Year": "year", "Pathogen": "pathogen"}) + # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. + tb = tb.format(["country", "year", "pathogen"]) + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=snap.metadata) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.py b/etl/steps/data/meadow/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.py new file mode 100644 index 00000000000..1f3c017f086 --- /dev/null +++ b/etl/steps/data/meadow/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.py @@ -0,0 +1,39 @@ +"""Load a snapshot and create a meadow dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("microbe_total_deaths_by_syndrome.csv") + + # Load data from snapshot. + tb = snap.read() + assert all(tb["Age"] == "All Ages") + assert all(tb["Sex"] == "Both sexes") + assert all(tb["Measure"] == "Deaths") + assert all(tb["Metric"] == "Number") + assert all(tb["Pathogen"] == "All pathogens") + assert all(tb["Counterfactual"] == "Total") + + # + # Process data. + tb = tb.drop(columns=["Age", "Sex", "Measure", "Metric", "Pathogen", "Counterfactual"]) + tb = tb.rename(columns={"Location": "country", "Year": "year"}) + # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. + tb = tb.format(["country", "year", "infectious_syndrome"]) + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=snap.metadata) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.py b/etl/steps/data/meadow/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.py new file mode 100644 index 00000000000..476dfee1a85 --- /dev/null +++ b/etl/steps/data/meadow/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.py @@ -0,0 +1,39 @@ +"""Load a snapshot and create a meadow dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("microbe_total_deaths_by_syndrome_amr.csv") + + # Load data from snapshot. + tb = snap.read() + assert all(tb["Age"] == "All Ages") + assert all(tb["Sex"] == "Both sexes") + assert all(tb["Measure"] == "Deaths") + assert all(tb["Metric"] == "Number") + assert all(tb["Pathogen"] == "All pathogens") + assert all(tb["Counterfactual"] == "Attributable") + + # + # Process data. + tb = tb.drop(columns=["Age", "Sex", "Measure", "Metric", "Pathogen", "Counterfactual"]) + tb = tb.rename(columns={"Location": "country", "Year": "year"}) + # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. + tb = tb.format(["country", "year", "infectious_syndrome"]) + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=snap.metadata) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/artificial_intelligence/2023-06-21/epoch.py b/etl/steps/data/meadow/artificial_intelligence/2023-06-21/epoch.py index 68f7786b5bf..30746a707c8 100644 --- a/etl/steps/data/meadow/artificial_intelligence/2023-06-21/epoch.py +++ b/etl/steps/data/meadow/artificial_intelligence/2023-06-21/epoch.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - import numpy as np from owid.catalog import Table from structlog import get_logger diff --git a/etl/steps/data/meadow/artificial_intelligence/2024-01-30/yougov_job_automation.py b/etl/steps/data/meadow/artificial_intelligence/2024-01-30/yougov_job_automation.py index 77f6d374f53..a5b9ed6b9e2 100644 --- a/etl/steps/data/meadow/artificial_intelligence/2024-01-30/yougov_job_automation.py +++ b/etl/steps/data/meadow/artificial_intelligence/2024-01-30/yougov_job_automation.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - import shared from structlog import get_logger diff --git a/etl/steps/data/meadow/artificial_intelligence/2024-01-30/yougov_robots.py b/etl/steps/data/meadow/artificial_intelligence/2024-01-30/yougov_robots.py index ce952025d2e..8bea38ec2cc 100644 --- a/etl/steps/data/meadow/artificial_intelligence/2024-01-30/yougov_robots.py +++ b/etl/steps/data/meadow/artificial_intelligence/2024-01-30/yougov_robots.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - import shared from structlog import get_logger diff --git a/etl/steps/data/meadow/artificial_intelligence/2024-12-05/epoch.py b/etl/steps/data/meadow/artificial_intelligence/2024-12-05/epoch.py new file mode 100644 index 00000000000..916eff1f4e7 --- /dev/null +++ b/etl/steps/data/meadow/artificial_intelligence/2024-12-05/epoch.py @@ -0,0 +1,73 @@ +"""Load a snapshot and create a meadow dataset.""" + +import numpy as np + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + paths.log.info("epoch.start") + + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("epoch.csv") + + # Read snapshot + tb = snap.read() + + # + # Process data. + # + # Define columns of interest. + cols = [ + "Model", + "Domain", + "Authors", + "Country (from Organization)", + "Organization", + "Organization categorization", + "Publication date", + "Parameters", + "Training compute (FLOP)", + "Training dataset size (datapoints)", + "Notability criteria", + ] + + # Check that the columns of interest are present + for col in cols: + assert col in tb.columns, f"Column '{col}' is missing from the dataframe." + + # Select the columns of interest + tb = tb[cols] + # Replace empty strings with NaN values + tb = tb.replace("", np.nan) + # Remove rows where all values are NaN + tb = tb.dropna(how="all") + + # Convert the training compute column to float + tb["Training compute (FLOP)"] = tb["Training compute (FLOP)"].astype(float) + + # Replace the missing values in the system column with the organization column. If organization column is NaN as well replace the missing values in the system column with the authors column + tb["Model"] = tb["Model"].fillna(tb["Organization"]).fillna(tb["Authors"]) + # Check that there are no NaN values in the system column + assert not tb["Model"].isna().any(), "NaN values found in 'Model' column after processing." + # + # Create a new table. + # + tb = tb.format(["model", "publication_date"]) + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=[tb], default_metadata=snap.metadata) + + # Save changes in the new garden dataset. + ds_meadow.save() + + paths.log.info("epoch.end") diff --git a/etl/steps/data/meadow/artificial_intelligence/2024-12-05/epoch_compute_intensive.py b/etl/steps/data/meadow/artificial_intelligence/2024-12-05/epoch_compute_intensive.py new file mode 100644 index 00000000000..a8509aef960 --- /dev/null +++ b/etl/steps/data/meadow/artificial_intelligence/2024-12-05/epoch_compute_intensive.py @@ -0,0 +1,66 @@ +"""Load a snapshot and create a meadow dataset.""" + +import numpy as np + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("epoch_compute_intensive.csv") + + # Load data from snapshot. + tb = snap.read() + + # + # Process data. + # + # Define columns of interest. + cols = [ + "Model", + "Domain", + "Authors", + "Country (from Organization)", + "Organization", + "Publication date", + "Parameters", + "Training compute (FLOP)", + "Training dataset size (datapoints)", + ] + + # Check that the columns of interest are present + for col in cols: + assert col in tb.columns, f"Column '{col}' is missing from the dataframe." + + # Select the columns of interest + tb = tb[cols] + # Replace empty strings with NaN values + tb = tb.replace("", np.nan) + # Remove rows where all values are NaN + tb = tb.dropna(how="all") + + # Convert the training compute column to float + tb["Training compute (FLOP)"] = tb["Training compute (FLOP)"].astype(float) + + # Replace the missing values in the system column with the organization column. If organization column is NaN as well replace the missing values in the system column with the authors column + tb["Model"] = tb["Model"].fillna(tb["Organization"]).fillna(tb["Authors"]) + # Check that there are no NaN values in the system column + assert not tb["Model"].isna().any(), "NaN values found in 'Model' column after processing." + # + # Create a new table. + # + tb = tb.format(["model", "publication_date"]) + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=snap.metadata) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/aviation_safety_network/2022-10-12/aviation_statistics.py b/etl/steps/data/meadow/aviation_safety_network/2022-10-12/aviation_statistics.py index e6583b346a4..4c9c3193378 100644 --- a/etl/steps/data/meadow/aviation_safety_network/2022-10-12/aviation_statistics.py +++ b/etl/steps/data/meadow/aviation_safety_network/2022-10-12/aviation_statistics.py @@ -2,6 +2,7 @@ Safety Network. """ + from typing import cast import pandas as pd diff --git a/etl/steps/data/meadow/climate/2024-01-31/ghg_concentration.py b/etl/steps/data/meadow/climate/2024-01-31/ghg_concentration.py index 1ca24557052..851a9ca4850 100644 --- a/etl/steps/data/meadow/climate/2024-01-31/ghg_concentration.py +++ b/etl/steps/data/meadow/climate/2024-01-31/ghg_concentration.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-01-31/hawaii_ocean_time_series.py b/etl/steps/data/meadow/climate/2024-01-31/hawaii_ocean_time_series.py index 0544b0cb638..0531bf1f1aa 100644 --- a/etl/steps/data/meadow/climate/2024-01-31/hawaii_ocean_time_series.py +++ b/etl/steps/data/meadow/climate/2024-01-31/hawaii_ocean_time_series.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-01-31/surface_temperature_analysis.py b/etl/steps/data/meadow/climate/2024-01-31/surface_temperature_analysis.py index 88791a644b7..ff64d875f6c 100644 --- a/etl/steps/data/meadow/climate/2024-01-31/surface_temperature_analysis.py +++ b/etl/steps/data/meadow/climate/2024-01-31/surface_temperature_analysis.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-03-11/ghg_concentration.py b/etl/steps/data/meadow/climate/2024-03-11/ghg_concentration.py index 1ca24557052..851a9ca4850 100644 --- a/etl/steps/data/meadow/climate/2024-03-11/ghg_concentration.py +++ b/etl/steps/data/meadow/climate/2024-03-11/ghg_concentration.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-03-11/hawaii_ocean_time_series.py b/etl/steps/data/meadow/climate/2024-03-11/hawaii_ocean_time_series.py index 0544b0cb638..0531bf1f1aa 100644 --- a/etl/steps/data/meadow/climate/2024-03-11/hawaii_ocean_time_series.py +++ b/etl/steps/data/meadow/climate/2024-03-11/hawaii_ocean_time_series.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-03-11/surface_temperature_analysis.py b/etl/steps/data/meadow/climate/2024-03-11/surface_temperature_analysis.py index 88791a644b7..ff64d875f6c 100644 --- a/etl/steps/data/meadow/climate/2024-03-11/surface_temperature_analysis.py +++ b/etl/steps/data/meadow/climate/2024-03-11/surface_temperature_analysis.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-04-17/ghg_concentration.py b/etl/steps/data/meadow/climate/2024-04-17/ghg_concentration.py index 1ca24557052..851a9ca4850 100644 --- a/etl/steps/data/meadow/climate/2024-04-17/ghg_concentration.py +++ b/etl/steps/data/meadow/climate/2024-04-17/ghg_concentration.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-04-17/hawaii_ocean_time_series.py b/etl/steps/data/meadow/climate/2024-04-17/hawaii_ocean_time_series.py index 0544b0cb638..0531bf1f1aa 100644 --- a/etl/steps/data/meadow/climate/2024-04-17/hawaii_ocean_time_series.py +++ b/etl/steps/data/meadow/climate/2024-04-17/hawaii_ocean_time_series.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-04-17/surface_temperature_analysis.py b/etl/steps/data/meadow/climate/2024-04-17/surface_temperature_analysis.py index 88791a644b7..ff64d875f6c 100644 --- a/etl/steps/data/meadow/climate/2024-04-17/surface_temperature_analysis.py +++ b/etl/steps/data/meadow/climate/2024-04-17/surface_temperature_analysis.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-05-20/ghg_concentration.py b/etl/steps/data/meadow/climate/2024-05-20/ghg_concentration.py index 1ca24557052..851a9ca4850 100644 --- a/etl/steps/data/meadow/climate/2024-05-20/ghg_concentration.py +++ b/etl/steps/data/meadow/climate/2024-05-20/ghg_concentration.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-05-20/hawaii_ocean_time_series.py b/etl/steps/data/meadow/climate/2024-05-20/hawaii_ocean_time_series.py index 0544b0cb638..0531bf1f1aa 100644 --- a/etl/steps/data/meadow/climate/2024-05-20/hawaii_ocean_time_series.py +++ b/etl/steps/data/meadow/climate/2024-05-20/hawaii_ocean_time_series.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-05-20/surface_temperature_analysis.py b/etl/steps/data/meadow/climate/2024-05-20/surface_temperature_analysis.py index 88791a644b7..ff64d875f6c 100644 --- a/etl/steps/data/meadow/climate/2024-05-20/surface_temperature_analysis.py +++ b/etl/steps/data/meadow/climate/2024-05-20/surface_temperature_analysis.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-07-23/ghg_concentration.py b/etl/steps/data/meadow/climate/2024-07-23/ghg_concentration.py index 1ca24557052..851a9ca4850 100644 --- a/etl/steps/data/meadow/climate/2024-07-23/ghg_concentration.py +++ b/etl/steps/data/meadow/climate/2024-07-23/ghg_concentration.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-07-23/hawaii_ocean_time_series.py b/etl/steps/data/meadow/climate/2024-07-23/hawaii_ocean_time_series.py index 0544b0cb638..0531bf1f1aa 100644 --- a/etl/steps/data/meadow/climate/2024-07-23/hawaii_ocean_time_series.py +++ b/etl/steps/data/meadow/climate/2024-07-23/hawaii_ocean_time_series.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-07-23/surface_temperature_analysis.py b/etl/steps/data/meadow/climate/2024-07-23/surface_temperature_analysis.py index 88791a644b7..ff64d875f6c 100644 --- a/etl/steps/data/meadow/climate/2024-07-23/surface_temperature_analysis.py +++ b/etl/steps/data/meadow/climate/2024-07-23/surface_temperature_analysis.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-09-30/ghg_concentration.py b/etl/steps/data/meadow/climate/2024-09-30/ghg_concentration.py index 1ca24557052..851a9ca4850 100644 --- a/etl/steps/data/meadow/climate/2024-09-30/ghg_concentration.py +++ b/etl/steps/data/meadow/climate/2024-09-30/ghg_concentration.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-09-30/hawaii_ocean_time_series.py b/etl/steps/data/meadow/climate/2024-09-30/hawaii_ocean_time_series.py index 0544b0cb638..0531bf1f1aa 100644 --- a/etl/steps/data/meadow/climate/2024-09-30/hawaii_ocean_time_series.py +++ b/etl/steps/data/meadow/climate/2024-09-30/hawaii_ocean_time_series.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-09-30/surface_temperature_analysis.py b/etl/steps/data/meadow/climate/2024-09-30/surface_temperature_analysis.py index 88791a644b7..ff64d875f6c 100644 --- a/etl/steps/data/meadow/climate/2024-09-30/surface_temperature_analysis.py +++ b/etl/steps/data/meadow/climate/2024-09-30/surface_temperature_analysis.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-11-18/ghg_concentration.py b/etl/steps/data/meadow/climate/2024-11-18/ghg_concentration.py index 1ca24557052..851a9ca4850 100644 --- a/etl/steps/data/meadow/climate/2024-11-18/ghg_concentration.py +++ b/etl/steps/data/meadow/climate/2024-11-18/ghg_concentration.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-11-18/hawaii_ocean_time_series.py b/etl/steps/data/meadow/climate/2024-11-18/hawaii_ocean_time_series.py index 0544b0cb638..0531bf1f1aa 100644 --- a/etl/steps/data/meadow/climate/2024-11-18/hawaii_ocean_time_series.py +++ b/etl/steps/data/meadow/climate/2024-11-18/hawaii_ocean_time_series.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate/2024-11-18/surface_temperature_analysis.py b/etl/steps/data/meadow/climate/2024-11-18/surface_temperature_analysis.py index 88791a644b7..ff64d875f6c 100644 --- a/etl/steps/data/meadow/climate/2024-11-18/surface_temperature_analysis.py +++ b/etl/steps/data/meadow/climate/2024-11-18/surface_temperature_analysis.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/climate_watch/2023-10-31/emissions_by_sector.py b/etl/steps/data/meadow/climate_watch/2023-10-31/emissions_by_sector.py index 4f6902d5d05..1afc4cb6bd3 100644 --- a/etl/steps/data/meadow/climate_watch/2023-10-31/emissions_by_sector.py +++ b/etl/steps/data/meadow/climate_watch/2023-10-31/emissions_by_sector.py @@ -1,4 +1,5 @@ """Load a snapshot and create a meadow dataset.""" + import gzip import json diff --git a/etl/steps/data/meadow/climate_watch/2024-11-21/emissions_by_sector.py b/etl/steps/data/meadow/climate_watch/2024-11-21/emissions_by_sector.py index 69072db0a22..cf98a8f1422 100644 --- a/etl/steps/data/meadow/climate_watch/2024-11-21/emissions_by_sector.py +++ b/etl/steps/data/meadow/climate_watch/2024-11-21/emissions_by_sector.py @@ -1,4 +1,5 @@ """Load a snapshot and create a meadow dataset.""" + import gzip import json diff --git a/etl/steps/data/meadow/covid/latest/sweden_covid.py b/etl/steps/data/meadow/covid/latest/sweden_covid.py index 6a3e3cc7975..e699ea2dbe3 100644 --- a/etl/steps/data/meadow/covid/latest/sweden_covid.py +++ b/etl/steps/data/meadow/covid/latest/sweden_covid.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/democracy/2024-05-22/eiu.py b/etl/steps/data/meadow/democracy/2024-05-22/eiu.py index 5c1513451f4..1e9b05f5941 100644 --- a/etl/steps/data/meadow/democracy/2024-05-22/eiu.py +++ b/etl/steps/data/meadow/democracy/2024-05-22/eiu.py @@ -1,4 +1,5 @@ """Load a snapshot and create a meadow dataset.""" + from owid.catalog.tables import Table, concat from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/meadow/demography/2024-11-26/multiple_births.py b/etl/steps/data/meadow/demography/2024-11-26/multiple_births.py new file mode 100644 index 00000000000..460a983484a --- /dev/null +++ b/etl/steps/data/meadow/demography/2024-11-26/multiple_births.py @@ -0,0 +1,32 @@ +"""Load a snapshot and create a meadow dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("multiple_births.7z") + + # Load data from snapshot. + tb = snap.read_in_archive("HMBD_pooled_data_30.09.2024.csv") + + # + # Process data. + # + # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. + tb = tb.format(["country", "year", "stillbirths"]) + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=snap.metadata) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/demography/2024-12-05/paternal_ages.py b/etl/steps/data/meadow/demography/2024-12-05/paternal_ages.py new file mode 100644 index 00000000000..051a93e721b --- /dev/null +++ b/etl/steps/data/meadow/demography/2024-12-05/paternal_ages.py @@ -0,0 +1,81 @@ +"""Load a snapshot and create a meadow dataset.""" + +import owid.catalog.processing as pr + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("paternal_ages.rdata") + + # Load data from snapshot. + tbs = snap.read_rda_multiple() + + # + # Process data. + # + # Read counts & rates + tb_counts = [tb.assign(tname=tname) for tname, tb in tbs.items() if tname.startswith("counts_")] + tb_counts = pr.concat(tb_counts, ignore_index=True) + tb_counts["code"] = tb_counts["tname"].str.split("_").str[1].str[:-1] + tb_rates = [tb.assign(tname=tname) for tname, tb in tbs.items() if tname.startswith("rates_")] + tb_rates = pr.concat(tb_rates, ignore_index=True) + tb_rates["code"] = tb_rates["tname"].str.split("_").str[1].str[:-1] + + # Drop duplicates + flag_nld = (tb_counts["tname"] == "counts_NLD4") & (tb_counts["year"] >= 1996) & (tb_counts["year"] <= 2014) + flag_dnk = (tb_counts["tname"] == "counts_DNK3") & (tb_counts["year"] >= 2007) & (tb_counts["year"] <= 2015) + flag_che = (tb_counts["tname"] == "counts_CHE4") & (tb_counts["year"] >= 2007) & (tb_counts["year"] <= 2014) + tb_counts = tb_counts.loc[~(flag_nld | flag_dnk | flag_che)] + + # Drop duplicates + flag_isl1 = (tb_rates["tname"] == "rates_ISL5") & (tb_rates["year"] >= 1981) & (tb_rates["year"] <= 2013) + flag_isl2 = (tb_rates["tname"] == "rates_ISL3") & (tb_rates["year"] >= 1981) + flag_gbr = (tb_rates["tname"] == "rates_GBREW4") & (tb_rates["year"] >= 1964) & (tb_rates["year"] <= 2013) + tb_rates = tb_rates.loc[~(flag_isl1 | flag_isl2 | flag_gbr)] + + # Dtypes + str_types = ["sourcelong", "tname", "GMI", "country", "type", "source"] + tb_counts = tb_counts.astype({col: "string" for col in str_types}) + tb_rates = tb_rates.astype({col: "string" for col in str_types}) + + # Drop spurious data + tb_counts = tb_counts.loc[tb_counts["source"] != "True"] + tb_rates = tb_rates.loc[~tb_rates["source"].isna()] + + # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. + tables = [ + tb_counts.format(["code", "year", "source", "type"], short_name="counts"), + tb_rates.format(["code", "year", "source", "type"], short_name="rates"), + ] + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset( + dest_dir, + tables=tables, + check_variables_metadata=True, + default_metadata=snap.metadata, + ) + + # Save changes in the new meadow dataset. + ds_meadow.save() + + +COLUMNS_COUNTS = [ + "country", + "year", + "type", + "mean age at childbirth as arithmetic mean", + "mean age at childbirth based demographic rates", + "source", +] diff --git a/etl/steps/data/meadow/demography/2024-12-06/shared.py b/etl/steps/data/meadow/demography/2024-12-06/shared.py new file mode 100644 index 00000000000..4095a690582 --- /dev/null +++ b/etl/steps/data/meadow/demography/2024-12-06/shared.py @@ -0,0 +1,372 @@ +"""Load a snapshot and create a meadow dataset. + +This snapshot step is a bit more complex than usual. This is because the snapshot is a ZIP file that contains numerous RDS files. These RDS files can be merged and concatenated, so that we build a single table with all the data. + + +The output table has index columns: country, year, scenario, sex, age, education. + +When values are aggregates, dimensions are set to "total". +""" + +import os +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + +import owid.catalog.processing as pr +import pyreadr +from owid.catalog import Table +from owid.catalog.tables import _add_table_and_variables_metadata_to_table + +from etl.helpers import PathFinder + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +# Renaming of relevant columns +COLUMNS_RENAME = { + "name": "country", + "period": "year", +} + +# Harmonization of the dimension values +REPLACE_AGE = { + "all": "total", +} +REPLACE_SEX = { + "both": "total", +} + + +def make_scenario_tables(tbs_scenario, tables_combine_edu, tables_concat, tables_drop, tables_composition): + """Create main table from all scenarios. + + Index: country, year, scenario, age, sex, education. + """ + # Obtain tables + tbs_base = [] + for scenario, tbs in tbs_scenario.items(): + paths.log.info(f"> scenario {scenario}") + tb = make_tables_from_scenario( + tbs=tbs, + scenario_num=scenario, + tables_combine_edu=tables_combine_edu, + tables_concat=tables_concat, + tables_drop=tables_drop, + tables_composition=tables_composition, + ) + tbs_base.append(tb) + + # Re-shape table structure + tbs_ = {group: [] for group in tables_composition} + for tbs in tbs_base: + for key, tb in tbs.items(): + tbs_[key].append(tb) + + return tbs_ + + +def concatenate_tables(tbs_scenario): + tables = [] + for tname, tbs in tbs_scenario.items(): + paths.log.info(f"Concatenating table {tname}") + # Check columns match for all tables in group + cols_index = None + for tb in tbs: + cols_index_ = _get_index_columns(tb) + if cols_index is None: + cols_index = list(cols_index_) + else: + assert set(cols_index) == set(cols_index_), "Unexpected index columns!" + # Merge + tb = pr.concat(tbs, short_name=tname) + + # Format + paths.log.info(f"Formatting table {tname}") + assert isinstance(cols_index, list) + tb = tb.format(cols_index + ["scenario"]) + + # Add to main list + tables.append(tb) + return tables + + +def make_tables_from_scenario(tbs, scenario_num, tables_combine_edu, tables_concat, tables_drop, tables_composition): + """Integrate all tables from scenario into single tables. + + We generate multiple single tables since different tables may use different dimensions. This helps in optimizing computation. + """ + # Sanity check country - country_code match + tables = [] + for tb in tbs: + # Check columns are in tables + assert ("country" in tb.columns) and ("country_code" in tb.columns), "Missing country or country_code!" + # Check there is a one-to-one correspondence + assert ( + tb.groupby("country")["country_code"].nunique().max() == 1 + ), "Multiple country codes for a single country!" + # Drop country_code + tb = tb.drop(columns="country_code") + tables.append(tb) + + # Create dictionary to ease navigation + tables = {t.m.short_name: t for t in tables} + # Dictionary with table combinations made + tables = reduce_tables(tables, tables_combine_edu, tables_concat, tables_drop) + + # Composition of tables + tables = consolidate_table_all(tables, scenario_num, tables_composition) + + return tables + + +def reduce_tables(tables, tables_combine_edu, tables_concat, tables_drop): + """Reduces the original number based on similar indicators. + + Given a key-value dictionary with all the tables, this function simplifies it structure: + + - It combines different tables into a single one. That is possible when, e.g. a table contains the same indicator but broken down by an additional dimension. + - Some tables don't have new data. These can be discarded. + """ + paths.log.info("Reducing tables...") + # Start by defining the output dictionary `tables_reduced`, with those tables that are not combined and thus should be kept, at least for now, as they are + tables_combined = [cc for c in tables_combine_edu for cc in c] + [cc for c in tables_concat for cc in c] + if ("net" in tables) and ("netedu" in tables): + tables_combined += ["net", "netedu"] + tables_not_combined = [name for name in tables.keys() if name not in tables_combined] + tables_reduced = {name: harmonize_tb(tables[name]) for name in tables_not_combined} + + # Iterate over the tables from `tables_combine_edu` to consolidate the pairs into single representation. + # Index 1 contains data broken down by 'education'. Index 0 contains data with 'education' = 'total'. + for tb_comb in tables_combine_edu: + # Load tables + tb1 = tables[tb_comb[0]] + tb2 = tables[tb_comb[1]] + # Drop in special case + if tb_comb[0] == "bpop": + tb1 = tb1.loc[tb1["age"] != "All"] + # Prepare tables for merge + tb1 = tb1.assign(education="total") + tb2 = tb2.rename(columns={tb_comb[1]: tb_comb[0]}) + # Check: columns are identical except 'education' + assert set(tb1.columns) == set(tb2.columns), "Unexpected columns!" + # Harmonize + tb1 = harmonize_tb(tb1) + tb2 = harmonize_tb(tb2) + # Concatenate + tb = pr.concat([tb1, tb2], ignore_index=True).drop_duplicates() + # Add to dictionary + tables_reduced[tb_comb[0]] = tb + for tb_comb in tables_concat: + # Load tables + tb1 = tables[tb_comb[0]] + tb2 = tables[tb_comb[1]] + # Prepare tables for merge + tb2 = tb2.rename(columns={tb_comb[1]: tb_comb[0]}) + # Check: columns are identical except 'education' + assert set(tb1.columns) == set(tb2.columns), "Unexpected columns!" + # Harmonize + tb1 = harmonize_tb(tb1) + tb2 = harmonize_tb(tb2) + # Concatenate + tb = pr.concat([tb1, tb2], ignore_index=True).drop_duplicates() + # Add to dictionary + tables_reduced[tb_comb[0]] = tb + + # Special case: net and netedu + if ("net" in tables) and ("netedu" in tables): + # net has age=all and sex=all, so we can drop these columns + # NOTE: net is equivalent to sum(netedu) + epsilon (probably unknown education?) + tb1 = tables["net"] + tb2 = tables["netedu"] + assert set(tb1["age"].unique()) == {"All"} + assert set(tb1["sex"].unique()) == {"Both"} + tb1 = tb1.drop(columns=["age", "sex"]) + tb1["education"] = "total" + # Rename + tb2 = tb2.rename(columns={"netedu": "net"}) + # Check: columns are identical except 'education' + assert set(tb1.columns) == set(tb2.columns), "Unexpected columns!" + # Harmonize + tb1 = harmonize_tb(tb1) + tb2 = harmonize_tb(tb2) + # Concatenate + tb = pr.concat([tb1, tb2], ignore_index=True).drop_duplicates() + tables_reduced["net"] = tb + + # Remove tables that are not needed + tables_reduced = {tname: tb for tname, tb in tables_reduced.items() if tname not in tables_drop} + + # Special: rename 'bpop' -> 'pop' + tables_reduced["pop"] = tables_reduced["bpop"].rename(columns={"bpop": "pop"}) + del tables_reduced["bpop"] + + # Sort dictionary by keys + tables_reduced = dict(sorted(tables_reduced.items())) + return tables_reduced + + +def consolidate_table_all(tables, scenario_num, tables_composition): + """Consolidate tables into new groups. + + Each table group is differentiated from the rest based on the index its tables use. + Grouping by index helps make the merge of all tables way faster. + The idea is that in garden, we process & export each table separately. + This function also harmonizes dimension names. + """ + # Group tables in new groups + tables_new = {} + for tname_new, tnames in tables_composition.items(): + paths.log.info(f"Building {tname_new}...") + # Get list with table objects, with harmonized dimensions + tbs_ = [] + # Index columns + cols_index = None + for tname in tnames: + # Get table + tb_ = tables[tname] + # Harmonize table + tb_ = harmonize_tb(tb_) + # Check dimensions + cols_index_ = _get_index_columns(tb_) + if cols_index is None: + cols_index = list(cols_index_) + else: + assert set(cols_index) == set(cols_index_), "Unexpected index columns!" + tbs_.append(tb_) + # Merge all tables in list + tb = merge_tables_opt(tbs_, on=cols_index, how="outer") + # Add scenario information + tb["scenario"] = scenario_num + tb["scenario"] = tb["scenario"].astype("string") + # Add consolidated table in main dictionary + tables_new[tname_new] = tb + return tables_new + + +def _get_index_columns(tb): + cols_index_all = ["country", "year", "sex", "age", "education"] + cols_index = list(tb.columns.intersection(cols_index_all)) + return cols_index + + +def harmonize_tb(tb): + """Harmonizes tables. + + - Dimensions are named differently in different tables. This function ensures that they are consistent. + - Makes sure DTypes are set correctly. + """ + if "age" in tb.columns: + tb["age"] = ( + tb["age"] + .str.lower() + .replace(REPLACE_AGE) + .str.replace("––", "-", regex=False) + .str.replace("--", "-", regex=False) + ).astype("string") + if "sex" in tb.columns: + tb["sex"] = tb["sex"].str.lower().replace(REPLACE_SEX).astype("string") + if "education" in tb.columns: + tb["education"] = tb["education"].str.lower().str.replace(" ", "_").astype("string") + + # Set dtype + tb = tb.astype( + { + "country": "string", + "year": "string", + } + ) + return tb + + +def merge_tables_opt(tables, **kwargs): + """Optimized & parallelized version of merge_tables.""" + + def _merge_pair(tables): + """Merge two tables.""" + left, right = tables + return left.merge(right, **kwargs) + + # Divide tables into pairs to merge in parallel + with ThreadPoolExecutor() as executor: + while len(tables) > 1: + # Pair tables and merge them in parallel + future_to_merge = { + executor.submit(_merge_pair, (tables[i], tables[i + 1])): i for i in range(0, len(tables) - 1, 2) + } + + # Collect merged tables + merged_tables = [] + for future in as_completed(future_to_merge): + merged_tables.append(future.result()) + + # If odd number of tables, append the last unpaired table + if len(tables) % 2 == 1: + merged_tables.append(tables[-1]) + + # Update tables list with merged results + tables = merged_tables + + # Final merged table + merged_tb = tables[0] + return merged_tb + + +def read_data_from_snap(snap, scenarios_expected): + """Read snapshot. + + Snapshot is a ZIP file that contains numerous RDS files. + """ + tbs_scenario = {} + with snap.extract_to_tempdir() as tmp: + files = os.listdir(tmp) + for i, f in enumerate(files): + if i % 10 == 0: + paths.log.info(f"Processing file {i}/{len(files)}: {f}") + # Add relevant columns + scenario = f.split("_")[0] + assert scenario in scenarios_expected, f"Unexpected scenario: {scenario}" + if scenario in {"22", "23"}: + continue + filename = f.split("_")[1] + # Read RDS file + path = tmp / Path(f) + if f.endswith(".rds"): + data = pyreadr.read_r(path) + assert set(data.keys()) == {None}, "Unexpected keys in RDS file!" + df = data[None] + # Map to table + tb = Table(df) + tb = _add_table_and_variables_metadata_to_table( + table=tb, metadata=snap.to_table_metadata(), origin=snap.metadata.origin + ) + tb.metadata.short_name = filename.replace(".rds", "") + elif f.endswith(".csv.gz"): + scenario = scenario.replace("ssp", "") + tb = pr.read_csv( + path, + metadata=snap.to_table_metadata(), + origin=snap.metadata.origin, + ) + tb.metadata.short_name = filename.replace(".csv.gz", "") + else: + raise ValueError(f"Unexpected file format: {f}!") + # Rename columns + tb = tb.rename(columns=COLUMNS_RENAME) + # Add to main list + if scenario in tbs_scenario.keys(): + tbs_scenario[scenario].append(tb) + else: + tbs_scenario[scenario] = [tb] + return tbs_scenario + + +def inspect_tbs_in_scenario(tbs): + """For exploration purposes only.""" + cols_index = ["country", "country_code", "scenario", "year", "age", "sex", "education"] + for tname, tb in tbs.items(): + cols = set(tb.columns) + cols_index_ = [col for col in cols_index if col in cols] + cols_indicators_ = [col for col in cols if col not in cols_index_] + # cols_ = list(sorted(cols_index_)) + list(sorted(cols_indicators_)) + print(f"> {', '.join(sorted(cols_indicators_))} ({tname}): {', '.join(cols_index_)}") + # print(f"{', '.join(cols_index_)}") diff --git a/etl/steps/data/meadow/demography/2024-12-06/wittgenstein_human_capital_historical.py b/etl/steps/data/meadow/demography/2024-12-06/wittgenstein_human_capital_historical.py new file mode 100644 index 00000000000..96da42c3798 --- /dev/null +++ b/etl/steps/data/meadow/demography/2024-12-06/wittgenstein_human_capital_historical.py @@ -0,0 +1,143 @@ +"""Load a snapshot and create a meadow dataset. + +This snapshot step is a bit more complex than usual. This is because the snapshot is a ZIP file that contains numerous RDS files. These RDS files can be merged and concatenated, so that we build a single table with all the data. + + +The output table has index columns: country, year, scenario, sex, age, education. + +When values are aggregates, dimensions are set to "total". +""" + +from etl.helpers import PathFinder, create_dataset + +from .shared import concatenate_tables, make_scenario_tables, read_data_from_snap + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +SCENARIOS_EXPECTED = { + "ssp1", + "ssp2", + "ssp3", + "ssp4", + "ssp5", +} + +# Renaming of relevant columns +COLUMNS_RENAME = { + "name": "country", + "period": "year", +} + +# Harmonization of the dimension values +REPLACE_AGE = { + "all": "total", +} +REPLACE_SEX = { + "both": "total", +} + + +# First table contains education = "all", the other the rest +TABLES_COMBINE_EDUCATION = [ + ("asfr", "easfr"), + ("assr", "eassr"), + # ("macb", "emacb"), + ("tfr", "etfr"), + ("bpop", "epop"), +] + +TABLES_CONCAT = [ + # ("prop", "bprop"), + ("mys", "bmys"), +] +TABLES_DROP = [] + +# Composition of tables +TABLES_COMPOSITION = { + # 0/ No dimension + "main": {"cbr", "cdr", "ggapmys15", "ggapmys25", "growth", "mage", "nirate", "odr", "ydr", "tdr", "macb"}, + # 1/ Sex dimension. NOTE: no sex=total + "by_sex": {"e0", "pryl15", "ryl15"}, + # 1/ Age dimension + "by_age": {"sexratio"}, + # 1/ Education dimension + "by_edu": {"ggapedu15", "ggapedu25", "tfr"}, + # 2/ Sex+Age dimensions. NOTE: no age=total + "by_sex_age": {"mys", "net"}, + # 2/ Age+Education dimensions. NOTE: no age=total, that's fine. We have tfr for all ages actually. + "by_age_edu": {"asfr"}, + # 3/ Sex+Age+Education dimensions + "by_sex_age_edu": {"assr", "pop"}, +} + + +def run(dest_dir: str) -> None: + """Overall, this step could take 6:30 minutes on my machine (lucas).""" + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("wittgenstein_human_capital_historical.zip") + + # Load data from snapshot. {"1": [tb1, tb2, ...], "2": [tb1, tb2, ...], ...} + # ~ 1:20 minutes (2 minutes if all scenarios are used) + tbs_scenario = read_data_from_snap(snap, SCENARIOS_EXPECTED) + + # + # Process data. + # + # Consolidate individual scenario tables: {"main": [tb_main1, tb_main2, ...], "by_sex": [tb_sex1, tb_sex2, ...], ...} + # ~ 3 minutes (4:30 if all scenarios are used) + # dix = {k: v for k, v in tbs_scenario.items() if k == "ssp1"} + # tbs_scenario_ = make_scenario_tables(dix) + tables = make_scenario_tables( + tbs_scenario=tbs_scenario, + tables_combine_edu=TABLES_COMBINE_EDUCATION, + tables_concat=TABLES_CONCAT, + tables_drop=TABLES_DROP, + tables_composition=TABLES_COMPOSITION, + ) + + # Filter out rows that are >2020 + year_ranges_ignore = [f"{i}-{i+5}" for i in range(2021, 2101, 5)] + [f"{i}.0-{i+5}.0" for i in range(2020, 2101, 5)] + year_ranges_ignore += [f"{i}.0" for i in range(2021, 2101)] + [f"{i}" for i in range(2020, 2101)] + + for tname, tbs in tables.items(): + for i, tb in enumerate(tbs): + # Remove unwanted years + tb = tb.loc[~(tb["year"].isin(year_ranges_ignore))] + # Remove duplicate rows + # An alternative to solve this is: drop age=All in bpop table. + # if tname == "by_sex_age_edu": + # cols = ["country", "year", "age", "education", "sex", "scenario"] + # x = tb.groupby(["country", "year", "age", "education", "sex", "scenario"], as_index=False).size() + # x = x[x["size"] > 1] + # assert set(x["year"].unique()) == {"2015"} + # assert set(x["age"].unique()) == {"total"} + # assert set(x["sex"].unique()) == {"male"} + # assert set(x["education"].unique()) == {"total"} + # tb = tb.drop_duplicates(subset=cols) + + # Overwrite + tables[tname][i] = tb + + # Remove duplicates + + # Concatenate: [table_main, table_sex, ...] + # ~ 2 minutes + tables = concatenate_tables(tables) + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset( + dest_dir, + tables=tables, + check_variables_metadata=True, + default_metadata=snap.metadata, + ) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/demography/2024-12-06/wittgenstein_human_capital_proj.py b/etl/steps/data/meadow/demography/2024-12-06/wittgenstein_human_capital_proj.py new file mode 100644 index 00000000000..861b9f98dbb --- /dev/null +++ b/etl/steps/data/meadow/demography/2024-12-06/wittgenstein_human_capital_proj.py @@ -0,0 +1,130 @@ +"""Load a snapshot and create a meadow dataset. + +This snapshot step is a bit more complex than usual. This is because the snapshot is a ZIP file that contains numerous RDS files. These RDS files can be merged and concatenated, so that we build a single table with all the data. + + +The output table has index columns: country, year, scenario, sex, age, education. + +When values are aggregates, dimensions are set to "total". +""" + +from etl.helpers import PathFinder, create_dataset + +from .shared import concatenate_tables, make_scenario_tables, read_data_from_snap + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +SCENARIOS_EXPECTED = { + "1", + "2", + "22", + "23", + "3", + "4", + "5", +} + +# Renaming of relevant columns +COLUMNS_RENAME = { + "name": "country", + "period": "year", +} + +# Harmonization of the dimension values +REPLACE_AGE = { + "all": "total", +} +REPLACE_SEX = { + "both": "total", +} + + +# First table contains education = "all", the other the rest +TABLES_COMBINE_EDUCATION = [ + ("asfr", "easfr"), + ("assr", "eassr"), + ("macb", "emacb"), + # ("net", "netedu"), + ("tfr", "etfr"), + ("bpop", "epop"), +] + +TABLES_CONCAT = [ + # ("prop", "bprop"), + ("mys", "bmys"), +] +TABLES_DROP = [ + # Several tables are just population! + "pop", + "pop-age", + "pop-age-edattain", + "pop-age-sex", + "pop-age-sex-edattain", + "pop-sex", + "pop-sex-edattain", + "pop-total", +] +# Composition of tables +TABLES_COMPOSITION = { + # 0/ No dimension + "main": {"cbr", "cdr", "emi", "ggapmys15", "ggapmys25", "growth", "imm", "mage", "nirate", "odr", "ydr", "tdr"}, + # 1/ Sex dimension. NOTE: no sex=total + "by_sex": {"e0", "pryl15", "ryl15"}, + # 1/ Age dimension + "by_age": {"sexratio"}, + # 1/ Education dimension + "by_edu": {"ggapedu15", "ggapedu25", "macb", "net", "tfr"}, + # 2/ Sex+Age dimensions. NOTE: no age=total + "by_sex_age": {"mys"}, + # 2/ Age+Education dimensions. NOTE: no age=total, that's fine. We have tfr for all ages actually. + "by_age_edu": {"asfr"}, + # 3/ Sex+Age+Education dimensions + "by_sex_age_edu": {"assr", "pop"}, +} + + +def run(dest_dir: str) -> None: + """Overall, this step could take 6:30 minutes on my machine (lucas).""" + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("wittgenstein_human_capital.zip") + + # Load data from snapshot. {"1": [tb1, tb2, ...], "2": [tb1, tb2, ...], ...} + # ~ 1:20 minutes (2 minutes if all scenarios are used) + tbs_scenario = read_data_from_snap(snap, SCENARIOS_EXPECTED) + + # + # Process data. + # + # Consolidate individual scenario tables: {"main": [tb_main1, tb_main2, ...], "by_sex": [tb_sex1, tb_sex2, ...], ...} + # ~ 3 minutes (4:30 if all scenarios are used) + # dix = {k: v for k, v in tbs_scenario.items() if k == "1"} + # tbs_scenario_ = make_scenario_tables(dix) + tbs_scenario = make_scenario_tables( + tbs_scenario=tbs_scenario, + tables_combine_edu=TABLES_COMBINE_EDUCATION, + tables_concat=TABLES_CONCAT, + tables_drop=TABLES_DROP, + tables_composition=TABLES_COMPOSITION, + ) + + # Concatenate: [table_main, table_sex, ...] + # ~ 2 minutes + tables = concatenate_tables(tbs_scenario) + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset( + dest_dir, + tables=tables, + check_variables_metadata=True, + default_metadata=snap.metadata, + ) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/education/2023-08-14/oecd_education.py b/etl/steps/data/meadow/education/2023-08-14/oecd_education.py index eba65ac3bb3..7e367c982af 100644 --- a/etl/steps/data/meadow/education/2023-08-14/oecd_education.py +++ b/etl/steps/data/meadow/education/2023-08-14/oecd_education.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/ember/2022-08-01/european_electricity_review.py b/etl/steps/data/meadow/ember/2022-08-01/european_electricity_review.py index a4d536ae608..48a34a2de06 100644 --- a/etl/steps/data/meadow/ember/2022-08-01/european_electricity_review.py +++ b/etl/steps/data/meadow/ember/2022-08-01/european_electricity_review.py @@ -1,6 +1,4 @@ -"""Meadow step for European Electricity Review (Ember, 2022). - -""" +"""Meadow step for European Electricity Review (Ember, 2022).""" from pathlib import Path from tempfile import TemporaryDirectory diff --git a/etl/steps/data/meadow/ember/2022-08-01/global_electricity_review.py b/etl/steps/data/meadow/ember/2022-08-01/global_electricity_review.py index b9fe1a47e41..8d20bed9eab 100644 --- a/etl/steps/data/meadow/ember/2022-08-01/global_electricity_review.py +++ b/etl/steps/data/meadow/ember/2022-08-01/global_electricity_review.py @@ -1,6 +1,4 @@ -"""Meadow step for Global Electricity Review (Ember, 2022). - -""" +"""Meadow step for Global Electricity Review (Ember, 2022).""" import pandas as pd from owid.catalog import Dataset, Table, TableMeta diff --git a/etl/steps/data/meadow/ember/2023-07-10/yearly_electricity.py b/etl/steps/data/meadow/ember/2023-07-10/yearly_electricity.py index 51543bc0054..df026f4f1fb 100644 --- a/etl/steps/data/meadow/ember/2023-07-10/yearly_electricity.py +++ b/etl/steps/data/meadow/ember/2023-07-10/yearly_electricity.py @@ -1,6 +1,5 @@ -"""Load snapshot of Ember's Yearly Electricity Data and create a raw data table. +"""Load snapshot of Ember's Yearly Electricity Data and create a raw data table.""" -""" from etl.helpers import PathFinder, create_dataset # Get naming conventions. diff --git a/etl/steps/data/meadow/ember/2023-12-12/european_electricity_review.py b/etl/steps/data/meadow/ember/2023-12-12/european_electricity_review.py index ec12f351a0d..438db18f2a3 100644 --- a/etl/steps/data/meadow/ember/2023-12-12/european_electricity_review.py +++ b/etl/steps/data/meadow/ember/2023-12-12/european_electricity_review.py @@ -1,6 +1,4 @@ -"""Meadow step for European Electricity Review (Ember, 2022). - -""" +"""Meadow step for European Electricity Review (Ember, 2022).""" from pathlib import Path from tempfile import TemporaryDirectory diff --git a/etl/steps/data/meadow/ember/2023-12-12/yearly_electricity.py b/etl/steps/data/meadow/ember/2023-12-12/yearly_electricity.py index 329d515d2c8..76dbc792ece 100644 --- a/etl/steps/data/meadow/ember/2023-12-12/yearly_electricity.py +++ b/etl/steps/data/meadow/ember/2023-12-12/yearly_electricity.py @@ -1,6 +1,5 @@ -"""Load snapshot of Ember's Yearly Electricity Data and create a raw data table. +"""Load snapshot of Ember's Yearly Electricity Data and create a raw data table.""" -""" from etl.helpers import PathFinder, create_dataset # Get naming conventions. diff --git a/etl/steps/data/meadow/ember/2024-05-08/yearly_electricity.py b/etl/steps/data/meadow/ember/2024-05-08/yearly_electricity.py index 11ccaed2eac..d93a4609ce1 100644 --- a/etl/steps/data/meadow/ember/2024-05-08/yearly_electricity.py +++ b/etl/steps/data/meadow/ember/2024-05-08/yearly_electricity.py @@ -1,6 +1,5 @@ -"""Load snapshot of Ember's Yearly Electricity Data and create a raw data table. +"""Load snapshot of Ember's Yearly Electricity Data and create a raw data table.""" -""" from etl.helpers import PathFinder, create_dataset # Get naming conventions. diff --git a/etl/steps/data/meadow/antibiotics/2024-11-20/bloodstream_amr.py b/etl/steps/data/meadow/ember/2024-11-20/european_wholesale_electricity_prices.py similarity index 79% rename from etl/steps/data/meadow/antibiotics/2024-11-20/bloodstream_amr.py rename to etl/steps/data/meadow/ember/2024-11-20/european_wholesale_electricity_prices.py index c938b4b7837..f8ae1613de9 100644 --- a/etl/steps/data/meadow/antibiotics/2024-11-20/bloodstream_amr.py +++ b/etl/steps/data/meadow/ember/2024-11-20/european_wholesale_electricity_prices.py @@ -11,16 +11,16 @@ def run(dest_dir: str) -> None: # Load inputs. # # Retrieve snapshot. - snap = paths.load_snapshot("bloodstream_amr.csv") + snap = paths.load_snapshot("european_wholesale_electricity_prices.csv") # Load data from snapshot. tb = snap.read() # # Process data. - tb = tb.rename(columns={"Location": "country", "Year": "year"}) + # # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. - tb = tb.format(["country", "year", "pathogen", "counterfactual"]) + tb = tb.format(["country", "date"]) # # Save outputs. diff --git a/etl/steps/data/meadow/emdat/2023-09-20/natural_disasters.py b/etl/steps/data/meadow/emdat/2023-09-20/natural_disasters.py index 5ef3c057326..720d6018096 100644 --- a/etl/steps/data/meadow/emdat/2023-09-20/natural_disasters.py +++ b/etl/steps/data/meadow/emdat/2023-09-20/natural_disasters.py @@ -1,6 +1,5 @@ -"""Load snapshot of EM-DAT natural disasters data and prepare a table with basic metadata. +"""Load snapshot of EM-DAT natural disasters data and prepare a table with basic metadata.""" -""" import warnings from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/meadow/emdat/2024-04-11/natural_disasters.py b/etl/steps/data/meadow/emdat/2024-04-11/natural_disasters.py index f7bf13921d0..5ea3cb54331 100644 --- a/etl/steps/data/meadow/emdat/2024-04-11/natural_disasters.py +++ b/etl/steps/data/meadow/emdat/2024-04-11/natural_disasters.py @@ -1,6 +1,5 @@ -"""Load snapshot of EM-DAT natural disasters data and prepare a table with basic metadata. +"""Load snapshot of EM-DAT natural disasters data and prepare a table with basic metadata.""" -""" from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/ess/2023-08-02/ess_trust.py b/etl/steps/data/meadow/ess/2023-08-02/ess_trust.py index 27b0aba56a1..04fbafe2b8e 100644 --- a/etl/steps/data/meadow/ess/2023-08-02/ess_trust.py +++ b/etl/steps/data/meadow/ess/2023-08-02/ess_trust.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - import owid.catalog.processing as pr from owid.catalog import Table from structlog import get_logger diff --git a/etl/steps/data/meadow/eurostat/2024-11-05/gas_and_electricity_prices.py b/etl/steps/data/meadow/eurostat/2024-11-05/gas_and_electricity_prices.py new file mode 100644 index 00000000000..071b4143ba3 --- /dev/null +++ b/etl/steps/data/meadow/eurostat/2024-11-05/gas_and_electricity_prices.py @@ -0,0 +1,78 @@ +"""Load a snapshot and create a meadow dataset.""" + +import zipfile + +import owid.catalog.processing as pr + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("gas_and_electricity_prices.zip") + + # Create a list to store each table. + tables = [] + # Open the ZIP file and read each TSV file. + with zipfile.ZipFile(snap.path, "r") as zip_file: + for file_name in zip_file.namelist(): + # Read each TSV file into a table. + with zip_file.open(file_name) as file: + dataset_code = file_name.split(".")[0] + # Each data file starts with comma-separated index columns, followed by tab-separated time data. + # Example: + # freq,product,nrg_cons,unit,tax,currency,geo\TIME_PERIOD 2007 2008... + # And for some datasets, there is annual data, and for others bi-annual data, e.g. 2007-S1 2007-S2 2008-S1... + # First, load this file as a table. + _tb = pr.read_csv( + file, sep=r",|\t", engine="python", metadata=snap.to_table_metadata(), origin=snap.metadata.origin + ) + # Identify index columns. + index_columns = [column for column in _tb.columns if not column[0].isdigit()] + # Melt the table to have a single "time" column. + _tb = _tb.melt(id_vars=index_columns, var_name="time", value_name="value") + # Remove spurious "TIME_PERIOD" from one of the columns. + _tb = _tb.rename(columns={column: column.replace("\\TIME_PERIOD", "") for column in _tb.columns}) + # Add the dataset code as a column. + _tb = _tb.assign(**{"dataset_code": dataset_code}) + # Append current table to the list. + tables.append(_tb) + + # Concatenate all tables. + tb = pr.concat(tables, ignore_index=True) + + # + # Process data. + # + # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. + tb = tb.format( + [ + "freq", + "product", + "nrg_cons", + "unit", + "tax", + "currency", + "geo", + "time", + "dataset_code", + "nrg_prc", + "customer", + "consom", + ] + ) + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/excess_mortality/latest/wmd.py b/etl/steps/data/meadow/excess_mortality/latest/wmd.py index 57e31272ad5..7538e233648 100644 --- a/etl/steps/data/meadow/excess_mortality/latest/wmd.py +++ b/etl/steps/data/meadow/excess_mortality/latest/wmd.py @@ -1,6 +1,7 @@ """Load a snapshot and create a meadow dataset. In this step we perform sanity checks on the expected input fields and the values that they take.""" + import pandas as pd from owid.catalog import Dataset, Table from structlog import get_logger diff --git a/etl/steps/data/meadow/excess_mortality/latest/xm_karlinsky_kobak.py b/etl/steps/data/meadow/excess_mortality/latest/xm_karlinsky_kobak.py index d8e5b339dcd..2d7e5b9ae64 100644 --- a/etl/steps/data/meadow/excess_mortality/latest/xm_karlinsky_kobak.py +++ b/etl/steps/data/meadow/excess_mortality/latest/xm_karlinsky_kobak.py @@ -1,6 +1,7 @@ """Load a snapshot and create a meadow dataset. In this step we perform sanity checks on the expected input fields and the values that they take.""" + from pathlib import Path from typing import List, Union diff --git a/etl/steps/data/meadow/faostat/2022-05-17/shared.py b/etl/steps/data/meadow/faostat/2022-05-17/shared.py index f772fa2ba5a..7fe899b2c0a 100644 --- a/etl/steps/data/meadow/faostat/2022-05-17/shared.py +++ b/etl/steps/data/meadow/faostat/2022-05-17/shared.py @@ -1,6 +1,4 @@ -"""Shared definitions in FAOSTAT meadow steps. - -""" +"""Shared definitions in FAOSTAT meadow steps.""" import os import tempfile diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_cahd.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_cahd.py index cc99efc2b22..cef3521be9c 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_cahd.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_cahd.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_cahd dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_ef.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_ef.py index c1b3ce5eec8..757b5e02430 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_ef.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_ef.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_ef dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_ei.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_ei.py index 8f8c520ac1c..4b0d4348390 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_ei.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_ei.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_ei dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_ek.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_ek.py index 8affbd5ac70..07c3f4309aa 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_ek.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_ek.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_ek dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_el.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_el.py index 7cda6b5ced7..d164df98484 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_el.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_el.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_el dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_emn.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_emn.py index e0341d5f29b..ea75d7c0fa2 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_emn.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_emn.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_emn dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_ep.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_ep.py index de1278faacf..9c82c265ac2 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_ep.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_ep.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_ep dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_esb.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_esb.py index d90d2c0538a..34d158e65e6 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_esb.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_esb.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_esb dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_fa.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_fa.py index 29014f1b54a..3e8f14ecaa7 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_fa.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_fa.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_fa dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_fbs.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_fbs.py index 65cbf54e4e3..4679d59bf79 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_fbs.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_fbs.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_fbs dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_fbsh.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_fbsh.py index ef0b7233357..a0a2333ad84 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_fbsh.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_fbsh.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_fbsh dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_fo.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_fo.py index 9932ebb4718..7cbcc918865 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_fo.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_fo.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_fo dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_fs.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_fs.py index 74f1892050e..a76c3fc43da 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_fs.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_fs.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_fs dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_gn.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_gn.py index 6cc1cdd3414..da4c69b9e62 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_gn.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_gn.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_gn dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_ic.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_ic.py index 76a7833c6f8..267297ac8d7 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_ic.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_ic.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_ic dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_lc.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_lc.py index a18b1892fbf..81d970ea86b 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_lc.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_lc.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_lc dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_qcl.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_qcl.py index d66b2edc113..e0b4ba83df6 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_qcl.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_qcl.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_qcl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_qi.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_qi.py index 460cc5faca5..5baac7a230c 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_qi.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_qi.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_qi dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_qv.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_qv.py index 07e74a4a95b..746aa1e2818 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_qv.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_qv.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_qv dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_rfb.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_rfb.py index ae439c21964..6d86069bc62 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_rfb.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_rfb.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_rfb dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_rfn.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_rfn.py index bae546a50e5..642ae04efb0 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_rfn.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_rfn.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_rfn dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_rl.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_rl.py index cb95f2263fb..7057af86a81 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_rl.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_rl.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_rl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_rp.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_rp.py index 010769e5587..19292b5d51e 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_rp.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_rp.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_rp dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_rt.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_rt.py index 7254a8063e9..08f4ac56b0e 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_rt.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_rt.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_rt dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_scl.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_scl.py index e9fc0ab99e4..6491839aea0 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_scl.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_scl.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_scl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_sdgb.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_sdgb.py index bde23c34c06..6c5486c50d9 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_sdgb.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_sdgb.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_sdgb dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_tcl.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_tcl.py index c5299c892af..9dff4b274a5 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_tcl.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_tcl.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_tcl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_ti.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_ti.py index 9cfc9f9af7a..b1a21e2f48b 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_ti.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_ti.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_ti dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2023-06-12/faostat_wcad.py b/etl/steps/data/meadow/faostat/2023-06-12/faostat_wcad.py index e3e3d84a0a8..85f48b13936 100644 --- a/etl/steps/data/meadow/faostat/2023-06-12/faostat_wcad.py +++ b/etl/steps/data/meadow/faostat/2023-06-12/faostat_wcad.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_wcad dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_cahd.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_cahd.py index cc99efc2b22..cef3521be9c 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_cahd.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_cahd.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_cahd dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_ei.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_ei.py index 8f8c520ac1c..4b0d4348390 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_ei.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_ei.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_ei dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_ek.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_ek.py index 8affbd5ac70..07c3f4309aa 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_ek.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_ek.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_ek dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_emn.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_emn.py index e0341d5f29b..ea75d7c0fa2 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_emn.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_emn.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_emn dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_esb.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_esb.py index d90d2c0538a..34d158e65e6 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_esb.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_esb.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_esb dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_fa.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_fa.py index 29014f1b54a..3e8f14ecaa7 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_fa.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_fa.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_fa dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_fbs.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_fbs.py index 65cbf54e4e3..4679d59bf79 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_fbs.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_fbs.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_fbs dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_fbsh.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_fbsh.py index ef0b7233357..a0a2333ad84 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_fbsh.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_fbsh.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_fbsh dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_fo.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_fo.py index 9932ebb4718..7cbcc918865 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_fo.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_fo.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_fo dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_fs.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_fs.py index 74f1892050e..a76c3fc43da 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_fs.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_fs.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_fs dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_ic.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_ic.py index 76a7833c6f8..267297ac8d7 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_ic.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_ic.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_ic dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_lc.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_lc.py index a18b1892fbf..81d970ea86b 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_lc.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_lc.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_lc dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_qcl.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_qcl.py index d66b2edc113..e0b4ba83df6 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_qcl.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_qcl.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_qcl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_qi.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_qi.py index 460cc5faca5..5baac7a230c 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_qi.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_qi.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_qi dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_qv.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_qv.py index 07e74a4a95b..746aa1e2818 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_qv.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_qv.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_qv dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_rfb.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_rfb.py index ae439c21964..6d86069bc62 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_rfb.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_rfb.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_rfb dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_rfn.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_rfn.py index bae546a50e5..642ae04efb0 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_rfn.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_rfn.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_rfn dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_rl.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_rl.py index cb95f2263fb..7057af86a81 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_rl.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_rl.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_rl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_rp.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_rp.py index 010769e5587..19292b5d51e 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_rp.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_rp.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_rp dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_rt.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_rt.py index 7254a8063e9..08f4ac56b0e 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_rt.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_rt.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_rt dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_scl.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_scl.py index e9fc0ab99e4..6491839aea0 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_scl.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_scl.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_scl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_sdgb.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_sdgb.py index bde23c34c06..6c5486c50d9 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_sdgb.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_sdgb.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_sdgb dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_tcl.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_tcl.py index c5299c892af..9dff4b274a5 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_tcl.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_tcl.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_tcl dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/faostat/2024-03-14/faostat_ti.py b/etl/steps/data/meadow/faostat/2024-03-14/faostat_ti.py index 9cfc9f9af7a..b1a21e2f48b 100644 --- a/etl/steps/data/meadow/faostat/2024-03-14/faostat_ti.py +++ b/etl/steps/data/meadow/faostat/2024-03-14/faostat_ti.py @@ -1,2 +1,3 @@ """FAOSTAT meadow step for faostat_ti dataset.""" + from .shared import run # noqa:F401 diff --git a/etl/steps/data/meadow/gapminder/2024-07-08/maternal_mortality.py b/etl/steps/data/meadow/gapminder/2024-07-08/maternal_mortality.py index 6bd017d45c6..07c6be353bf 100644 --- a/etl/steps/data/meadow/gapminder/2024-07-08/maternal_mortality.py +++ b/etl/steps/data/meadow/gapminder/2024-07-08/maternal_mortality.py @@ -64,9 +64,9 @@ def run(dest_dir: str) -> None: tb.loc[(tb["year"] == "1967") & (tb["Country"] == "Sweden") & (tb["Maternal deaths"] == 39), "year"] = "1957" # wrong entry for US (duplicate 1967 -> should be 1957) - tb.loc[ - (tb["year"] == "1967") & (tb["Country"] == "United States") & (tb["Live Births"] == 4308000), "year" - ] = "1957" + tb.loc[(tb["year"] == "1967") & (tb["Country"] == "United States") & (tb["Live Births"] == 4308000), "year"] = ( + "1957" + ) # wrong entry for Belgium (duplicate 1973 -> should be 1873) tb.loc[(tb["year"] == "1973") & (tb["Country"] == "Belgium") & (tb["Maternal deaths"] == 1283), "year"] = "1873" diff --git a/etl/steps/data/meadow/happiness/2012-03-20/happiness.py b/etl/steps/data/meadow/happiness/2012-03-20/happiness.py index 8fa79c6d2e8..10f3f831307 100644 --- a/etl/steps/data/meadow/happiness/2012-03-20/happiness.py +++ b/etl/steps/data/meadow/happiness/2012-03-20/happiness.py @@ -1,4 +1,5 @@ """Load a snapshot and create a meadow dataset.""" + from datetime import datetime from typing import cast diff --git a/etl/steps/data/meadow/happiness/2023-03-20/happiness.py b/etl/steps/data/meadow/happiness/2023-03-20/happiness.py index d1196f03f26..f0709af3cd8 100644 --- a/etl/steps/data/meadow/happiness/2023-03-20/happiness.py +++ b/etl/steps/data/meadow/happiness/2023-03-20/happiness.py @@ -1,4 +1,5 @@ """Load a snapshot and create a meadow dataset.""" + from datetime import datetime import pandas as pd diff --git a/etl/steps/data/meadow/harvard/2023-09-18/colonial_dates_dataset.py b/etl/steps/data/meadow/harvard/2023-09-18/colonial_dates_dataset.py index 8f04fd39fd0..cb15d8baf9a 100644 --- a/etl/steps/data/meadow/harvard/2023-09-18/colonial_dates_dataset.py +++ b/etl/steps/data/meadow/harvard/2023-09-18/colonial_dates_dataset.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/hmd/2022-12-07/life_tables.py b/etl/steps/data/meadow/hmd/2022-12-07/life_tables.py index 064aa20471f..d50ea06c9f6 100644 --- a/etl/steps/data/meadow/hmd/2022-12-07/life_tables.py +++ b/etl/steps/data/meadow/hmd/2022-12-07/life_tables.py @@ -28,6 +28,7 @@ ... """ + import os import re import tempfile diff --git a/etl/steps/data/meadow/hmd/2024-11-19/hfd.py b/etl/steps/data/meadow/hmd/2024-11-19/hfd.py index b785f4c433a..01c6711e216 100644 --- a/etl/steps/data/meadow/hmd/2024-11-19/hfd.py +++ b/etl/steps/data/meadow/hmd/2024-11-19/hfd.py @@ -215,6 +215,7 @@ 2y totbirthsRRbo """ + from pathlib import Path import owid.catalog.processing as pr diff --git a/etl/steps/data/meadow/hmd/2024-12-01/hmd.py b/etl/steps/data/meadow/hmd/2024-12-01/hmd.py new file mode 100644 index 00000000000..d4a56860adf --- /dev/null +++ b/etl/steps/data/meadow/hmd/2024-12-01/hmd.py @@ -0,0 +1,325 @@ +"""Load a snapshot and create a meadow dataset.""" + +import re +from io import StringIO +from pathlib import Path +from typing import List + +import owid.catalog.processing as pr +from owid.catalog import Table + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +# Life tables +FOLDERS_LT = [ + "lt_male", + "lt_female", + "lt_both", + "c_lt_male", + "c_lt_female", + "c_lt_both", +] +REGEX_LT = ( + r"(?P[a-zA-Z\-\s,]+), Life tables \((?P[a-zA-Z]+) (?P\d+x\d+)\), (?P[a-zA-Z]+)" + r"\tLast modified: (?P\d+ [a-zA-Z]{3} \d+); Methods Protocol: v\d+ \(\d+\)\n\n(?P(?s:.)*)" +) +COLUMNS_RENAME_LT = { + "mx": "central_death_rate", + "qx": "probability_of_death", + "ax": "average_survival_length", + "lx": "number_survivors", + "dx": "number_deaths", + "Lx": "number_person_years_lived", + "Tx": "number_person_years_remaining", + "ex": "life_expectancy", +} + +# Exposures +FOLDERS_EXPOSURES = [ + "c_exposures", + "exposures", +] +REGEX_EXP = ( + r"(?P[a-zA-Z\-\s,]+), (?PExposure) to risk \((?P[a-zA-Z]+) (?P\d+x\d+)\),\s\tLast modified: " + r"(?P\d+ [a-zA-Z]{3} \d+); Methods Protocol: v\d+ \(\d+\)\n\n(?P(?s:.)*)" +) + +# Mortality +FOLDERS_MOR = [ + "deaths", +] +REGEX_MOR = ( + r"(?P[a-zA-Z\-\s,]+), (?PDeaths) \((?P[a-zA-Z]+) (?P\d+x\d+|Lexis triangle)\),\s\tLast modified: " + r"(?P\d+ [a-zA-Z]{3} \d+); Methods Protocol: v\d+ \(\d+\)\n\n(?P(?s:.)*)" +) +# Population +FOLDERS_POP = [ + "population", +] +REGEX_POP = ( + r"(?P[a-zA-Z\-\s,]+?),?\s?(?PPopulation) size \((?P1\-year|abridged)\)\s+Last modified: " + r"(?P\d+ [a-zA-Z]{3} \d+)(; Methods Protocol: v\d+ \(\d+\)|,MPv\d \(in development\))\n\n(?P(?s:.)*)" +) +# Births +FOLDERS_BIRTHS = [ + "births", +] +REGEX_BIRTHS = ( + r"(?P[a-zA-Z\-\s,]+),\s+(?PBirths) \((?P1\-year)\)\s+Last modified: " + r"(?P\d+ [a-zA-Z]{3} \d+); Methods Protocol: v\d+ \(\d+\)\n\n(?P(?s:.)*)" +) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("hmd.zip") + + # Load data from snapshot. + with snap.extract_to_tempdir() as tmpdir: + # Population + tb_pop = make_tb( + path=Path(tmpdir), + main_folders=FOLDERS_POP, + regex=REGEX_POP, + snap=snap, + ) + + # Life tables + tb_lt = make_tb( + path=Path(tmpdir), + main_folders=FOLDERS_LT, + regex=REGEX_LT, + snap=snap, + ) + # Exposure + tb_exp = make_tb( + path=Path(tmpdir), + main_folders=FOLDERS_EXPOSURES, + regex=REGEX_EXP, + snap=snap, + ) + # Mortality + tb_m = make_tb( + path=Path(tmpdir), + main_folders=FOLDERS_MOR, + regex=REGEX_MOR, + snap=snap, + ) + + # Births + tb_bi = make_tb( + path=Path(tmpdir), + main_folders=FOLDERS_BIRTHS, + regex=REGEX_BIRTHS, + snap=snap, + ) + + # Life tables + ## Column rename + ## e.g. "Lx -> lx" and "lx -> lx". This will cause an error when setting the index. + tb_lt = tb_lt.rename(columns=COLUMNS_RENAME_LT) + + # Population + ## Invert 'abridged' <-> '1-year' in the type column + message = "Types 'abridged' and '1-year' might not be reversed anymore!" + assert not tb_pop.loc[tb_pop["format"] == "abridged", "Age"].str.contains("-").any(), message + assert tb_pop.loc[tb_pop["format"] == "1-year", "Age"].str.contains("80-84").any(), message + tb_pop["format"] = tb_pop["format"].map( + lambda x: "1-year" if x == "abridged" else "abridged" if x == "1-year" else x + ) + + # Check missing values + _check_nas(tb_lt, 0.01, 14) + _check_nas(tb_exp, 0.23, 47) + _check_nas(tb_m, 0.001, 1) + _check_nas(tb_pop, 0.001, 1) + + # Ensure correct year dtype + tb_lt = _clean_year(tb_lt) + tb_exp = _clean_year(tb_exp) + tb_m = _clean_year(tb_m) + tb_bi = _clean_year(tb_bi) + tb_pop = _clean_population_type(tb_pop) + + # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. + tables = [ + tb_lt.format(["country", "year", "sex", "age", "type", "format"], short_name="life_tables"), + tb_exp.format(["country", "year", "sex", "age", "type", "format"], short_name="exposures"), + tb_m.format(["country", "year", "sex", "age", "type", "format"], short_name="deaths"), + tb_pop.format(["country", "year", "sex", "age", "format"], short_name="population"), + tb_bi.format(["country", "year", "sex", "format"], short_name="births"), + ] + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset( + dest_dir, + tables=tables, + check_variables_metadata=True, + default_metadata=snap.metadata, + ) + + # Save changes in the new meadow dataset. + ds_meadow.save() + + +def make_tb(path: Path, main_folders: List[str], regex: str, snap) -> Table: + """Create table from multiple category folders. + + It inspects the content in `main_folders` (should be in `path`), and looks for TXT files to parse into tables. + + The output is a table with the relevant indicators and dimensions for all the categories. + + Arguments: + path: Path where the HMD export is located. + main_folders: List of folders to consider in `path`. These should typically be categories, which + group different individual indicators + regex: Regex to extract the metadata for a set of TXTs file found in main_folders. We need this + because the structure of the header in the TXT files slightly varies depending on + the indicator. + """ + # List with all relevant tables + tbs = [] + # Iterate over each top-level folder + for category_folder in main_folders: + main_folder_path = path / category_folder + if not main_folder_path.is_dir(): + raise FileNotFoundError(f"Folder {main_folder_path} not found in {path}") + # Iterate over each indicator folder + for indicator_path in main_folder_path.iterdir(): + if "lexis" in indicator_path.name: + continue + if indicator_path.is_dir(): + # Read all TXT files in the indicator folder, and put them as a single table + paths.log.info(f"Creating list of tables from available files in {path}...") + files = list(indicator_path.glob("*.txt")) + tbs_ = [make_tb_from_txt(f, regex, snap) for f in files] + tbs.extend(tbs_) + # Concatenate all dataframes + tb = pr.concat(tbs, ignore_index=True) + return tb + + +def make_tb_from_txt(text_path: Path, regex: str, snap) -> Table: + """Create a table from a TXT file.""" + # print(text_path) + # Extract fields + groups = extract_fields(regex, text_path) + + # Build df + tb = parse_table(groups["data"], snap) + + # Optional melt + if ("Female" in tb.columns) and ("Male" in tb.columns): + id_vars = [col for col in ["Age", "Year"] if col in tb.columns] + if "name" not in groups: + raise ValueError( + f"Indicator name not found in {text_path}! Please revise that source files' content matches FILE_REGEX." + ) + tb = tb.melt(id_vars=id_vars, var_name="sex", value_name=groups["name"]) + + # Add dimensions + tb = tb.assign( + country=groups["country"], + ) + + # Optional sex column + if "sex" in groups: + tb["sex"] = groups["sex"] + if "format" in groups: + tb["format"] = groups["format"] + if "type" in groups: + tb["type"] = groups["type"] + return tb + + +def extract_fields(regex: str, path: Path) -> dict: + """Structure the fields in the raw TXT file.""" + # Read single file + with open(path, "r") as f: + text = f.read() + # Get relevant fields + match = re.search(regex, text) + if match is not None: + groups = match.groupdict() + else: + raise ValueError(f"No match found in {f}! Please revise that source files' content matches FILE_REGEX.") + return groups + + +def parse_table(data_raw: str, snap): + """Given the raw data from the TXT file (as string) map it to a table.""" + tb_str = data_raw.strip() + tb_str = re.sub(r"\n\s+", "\n", tb_str) + tb_str = re.sub(r"[^\S\r\n]+", "\t", string=tb_str) + tb = pr.read_csv( + StringIO(tb_str), + sep="\t", + na_values=["."], + metadata=snap.to_table_metadata(), + origin=snap.m.origin, + ) + + return tb + + +def _check_nas(tb, missing_row_max, missing_countries_max): + """Check missing values & countries in data.""" + row_nans = tb.isna().any(axis=1) + assert ( + row_nans.sum() / len(tb) < missing_row_max + ), f"Too many missing values in life tables: {row_nans.sum()/len(tb)}" + + # Countries missing + countries_missing_data = tb.loc[row_nans, "country"].unique() + assert ( + len(countries_missing_data) / len(tb) < missing_countries_max + ), f"Too many missing values in life tables: {len(countries_missing_data)}" + + +def _clean_population_type(tb): + """Data provider notes the following: + + For populations with territorial changes, two sets of population estimates are given for years in which a territorial change occurred. The first set of estimates (identified as year "19xx-") refers to the population just before the territorial change, whereas the second set (identified as year "19xx+") refers to the population just after the change. For example, in France, the data for "1914-" cover the previous territory (i.e., as of December 31, 1913), whereas the data for "1914+" reflect the territorial boundaries as of January 1, 1914. + + To avoid confusion and duplicity, whenever there are multiple entries for a year, we keep YYYY+ definition for the year (e.g. country with new territorial changes). + """ + # Crete new column with the year. + regex = r"\b\d{4}\b" + tb["year"] = tb["Year"].astype("string").str.extract(f"({regex})", expand=False) + assert tb["year"].notna().all(), "Year extraction was successful!" + tb["year"] = tb["year"].astype(int) + + # Ensure raw year is as expected + assert ( + tb.groupby(["country", "year", "Age", "sex", "format"]).Year.nunique().max() == 2 + ), "Unexpected number of years (+/-)" + + # Drop duplicate years, keeping YYYY+. + tb["Year"] = tb["Year"].astype("string") + tb = tb.sort_values("Year") + tb = tb.drop_duplicates(subset=["year", "Age", "sex", "country", "format"], keep="first").drop(columns="Year") + + tb = tb.rename(columns={"year": "Year"}) + + # Additionally, remove year periods + tb = _clean_year(tb) + + return tb + + +def _clean_year(tb): + # Remove year ranges, and convert to int + flag = tb["Year"].astype("string").str.contains("-") + tb = tb.loc[~flag] + tb["Year"] = tb["Year"].astype("int") + return tb diff --git a/etl/steps/data/meadow/hmd/2024-12-03/hmd_country.py b/etl/steps/data/meadow/hmd/2024-12-03/hmd_country.py new file mode 100644 index 00000000000..ef50429e65c --- /dev/null +++ b/etl/steps/data/meadow/hmd/2024-12-03/hmd_country.py @@ -0,0 +1,69 @@ +"""Load a snapshot and create a meadow dataset.""" + +from pathlib import Path + +import owid.catalog.processing as pr + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("hmd_country.zip") + + # Load data from snapshot. + paths.log.info("Loading data from snapshot.") + tbs = [] + with snap.extract_to_tempdir() as tmp_dir: + p = Path(tmp_dir) + files = p.glob("**/InputDB/*month.txt") + for f in files: + tb_ = pr.read_csv( + f, + na_values=["."], + metadata=snap.to_table_metadata(), + origin=snap.m.origin, + ) + tb_.columns = tb_.columns.str.strip() + tb_ = tb_.rename( + columns={ + "NoteCode1": "Note1", + "NoteCode2": "Note2", + "NoteCode3": "Note3", + } + ) + tbs.append(tb_) + + # Concatenate + paths.log.info("Concatenating tables.") + tb = pr.concat(tbs, ignore_index=True) + tb = tb.rename(columns={"PopName": "country"}) + + # + # Process data. + # + paths.log.info("Processing data.") + tb = tb.groupby(["country", "Year", "Month"], as_index=False)["Births"].mean() + tb = tb.astype( + { + "country": "string", + "Month": "string", + } + ) + # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. + tb = tb.format(["country", "year", "month"], short_name="monthly") + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=snap.metadata) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/iea/2024-11-20/fossil_fuel_subsidies.py b/etl/steps/data/meadow/iea/2024-11-20/fossil_fuel_subsidies.py new file mode 100644 index 00000000000..85eb9491ead --- /dev/null +++ b/etl/steps/data/meadow/iea/2024-11-20/fossil_fuel_subsidies.py @@ -0,0 +1,128 @@ +"""Load a snapshot and create a meadow dataset.""" + +import owid.catalog.processing as pr +from owid.catalog import Table + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +# Assumed USD year. +DOLLAR_YEAR = 2023 + + +def prepare_subsidies_by_country_table(tb_subsidies: Table) -> Table: + # The table is split into two subtables: Above, global data, and below, country data. They are separated by an empty row. + columns_total = tb_subsidies.loc[0].tolist() + table_country_start_index = tb_subsidies[tb_subsidies[tb_subsidies.columns[0]] == "Country"].index[0] + columns_countries = tb_subsidies.loc[table_country_start_index].tolist() + error = "Subsidies by country sheet has changed." + assert tb_subsidies.columns[0] == f"Unit: Real {DOLLAR_YEAR} million USD", error + # Check that tables are aligned. + assert columns_total[2:] == columns_countries[2:], error + # Rename columns. + columns = columns_countries[0:2] + [str(int(year)) for year in columns_countries[2:]] + # Extract global subtable and assign country "World". + tb_global = tb_subsidies.loc[1 : table_country_start_index - 1].dropna(how="all").reset_index(drop=True) + tb_global = tb_global.rename( + columns={old_column: new_column for old_column, new_column in zip(tb_global.columns, columns)}, errors="raise" + ) + tb_global["Country"] = "World" + tb_global["Product"] = tb_global["Product"].replace({"All Products": "Total", "Natural Gas": "Gas"}) + # Extract countries subtable. + tb_countries = tb_subsidies.loc[table_country_start_index + 1 :].reset_index(drop=True) + tb_countries = tb_countries.rename( + columns={old_column: new_column for old_column, new_column in zip(tb_countries.columns, columns)}, + errors="raise", + ) + # Combine both tables. + tb = pr.concat([tb_global, tb_countries], ignore_index=True) + # Transpose table. + tb = tb.melt(id_vars=["Country", "Product"], var_name="Year", value_name="subsidy") + + # Improve format. + tb = tb.format(["country", "year", "product"]) + + return tb + + +def prepare_indicators_by_country_table(tb_indicators: Table) -> Table: + # The year of the data is given in the very first cell. The actual table starts a few rows below. + error = "Indicators by country sheet has changed." + assert tb_indicators.columns[0] == f"Indicators for year {DOLLAR_YEAR}", error + columns = { + "Country": "country", + "Average subsidisation rate (%)": "subsidization_rate", + "Subsidy per capita ($/person)": "subsidy_per_capita", + "Total subsidy as share of GDP (%)": "subsidy_as_share_of_gdp", + } + assert tb_indicators.loc[2].tolist() == list(columns), error + tb_indicators = tb_indicators.loc[3:].reset_index(drop=True) + tb_indicators = tb_indicators.rename( + columns={old_column: new_column for old_column, new_column in zip(tb_indicators.columns, columns.values())}, + errors="raise", + ) + # Add a year column. + tb_indicators = tb_indicators.assign(**{"year": DOLLAR_YEAR}) + # Improve format. + tb_indicators = tb_indicators.format(short_name="fossil_fuel_subsidies_indicators") + + return tb_indicators + + +def prepare_transport_oil_table(tb_transport: Table) -> Table: + columns = ["country"] + [str(int(year)) for year in tb_transport.loc[0][1:].tolist()] + error = "Transport Oil Subsidies sheet has changed." + assert tb_transport.columns[0] == f"Unit: Real {DOLLAR_YEAR} million USD", error + assert [column.isdigit() for column in columns[1:]], error + tb_transport = tb_transport.loc[1:].reset_index(drop=True) + tb_transport = tb_transport.rename( + columns={old_column: new_column for old_column, new_column in zip(tb_transport.columns, columns)}, + errors="raise", + ) + # Transpose table. + tb_transport = tb_transport.melt(id_vars=["country"], var_name="year", value_name="transport_oil_subsidy") + # Improve format. + tb_transport = tb_transport.format(short_name="fossil_fuel_subsidies_transport_oil") + + return tb_transport + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("fossil_fuel_subsidies.xlsx") + + # Load data from all relevant sheets in the snapshot file. + tb_subsidies = snap.read(sheet_name="Subsidies by country", skiprows=3) + tb_indicators = snap.read(sheet_name="Indicators by country") + tb_transport = snap.read(sheet_name="Transport Oil Subsidies", skiprows=3) + + # + # Process data. + # + # Prepare "Subsidies by country" table. + tb_subsidies = prepare_subsidies_by_country_table(tb_subsidies=tb_subsidies) + + # Prepare "Indicators by country" table. + tb_indicators = prepare_indicators_by_country_table(tb_indicators=tb_indicators) + + # Prepare "Transport Oil Subsidies" table. + tb_transport = prepare_transport_oil_table(tb_transport=tb_transport) + + # + # Save outputs. + # + # Create a new meadow dataset. + ds_meadow = create_dataset( + dest_dir, + tables=[tb_subsidies, tb_indicators, tb_transport], + check_variables_metadata=True, + default_metadata=snap.metadata, + ) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/igh/2024-07-05/better_data_homelessness.py b/etl/steps/data/meadow/igh/2024-07-05/better_data_homelessness.py index 6d5ff6be715..8bc3663e5c7 100644 --- a/etl/steps/data/meadow/igh/2024-07-05/better_data_homelessness.py +++ b/etl/steps/data/meadow/igh/2024-07-05/better_data_homelessness.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/ihme_gbd/2024-07-02/gbd_healthy_life_expectancy.py b/etl/steps/data/meadow/ihme_gbd/2024-07-02/gbd_healthy_life_expectancy.py index 3bc912e841e..7898ad6b4ab 100644 --- a/etl/steps/data/meadow/ihme_gbd/2024-07-02/gbd_healthy_life_expectancy.py +++ b/etl/steps/data/meadow/ihme_gbd/2024-07-02/gbd_healthy_life_expectancy.py @@ -1,4 +1,5 @@ """Load a snapshot and create a meadow dataset.""" + import zipfile from owid.catalog import Table diff --git a/etl/steps/data/meadow/ihme_gbd/2024-07-02/gbd_life_expectancy.py b/etl/steps/data/meadow/ihme_gbd/2024-07-02/gbd_life_expectancy.py index f584c0f926e..0dd79d04fe9 100644 --- a/etl/steps/data/meadow/ihme_gbd/2024-07-02/gbd_life_expectancy.py +++ b/etl/steps/data/meadow/ihme_gbd/2024-07-02/gbd_life_expectancy.py @@ -1,4 +1,5 @@ """Load a snapshot and create a meadow dataset.""" + import zipfile from owid.catalog import Table diff --git a/etl/steps/data/meadow/irena/2023-12-12/renewable_energy_patents.py b/etl/steps/data/meadow/irena/2023-12-12/renewable_energy_patents.py index 44ec7c4668c..40639318b86 100644 --- a/etl/steps/data/meadow/irena/2023-12-12/renewable_energy_patents.py +++ b/etl/steps/data/meadow/irena/2023-12-12/renewable_energy_patents.py @@ -1,6 +1,5 @@ -"""Load snapshot of IRENA's Renewable Energy Patents and create a raw data table. +"""Load snapshot of IRENA's Renewable Energy Patents and create a raw data table.""" -""" from etl.helpers import PathFinder, create_dataset # Get naming conventions. diff --git a/etl/steps/data/meadow/language/2024-07-17/ethnologue.py b/etl/steps/data/meadow/language/2024-07-17/ethnologue.py index 1958a9492a3..44ff142e03c 100644 --- a/etl/steps/data/meadow/language/2024-07-17/ethnologue.py +++ b/etl/steps/data/meadow/language/2024-07-17/ethnologue.py @@ -1,4 +1,5 @@ """Load a snapshot and create a meadow dataset.""" + import zipfile from owid.catalog import processing as pr diff --git a/etl/steps/data/meadow/lis/2024-06-13/luxembourg_income_study.py b/etl/steps/data/meadow/lis/2024-06-13/luxembourg_income_study.py index 7e10a215d88..4b3daa7a566 100644 --- a/etl/steps/data/meadow/lis/2024-06-13/luxembourg_income_study.py +++ b/etl/steps/data/meadow/lis/2024-06-13/luxembourg_income_study.py @@ -3,7 +3,6 @@ Country names are converted from iso-2 codes in this step and years are reformated from "YY" to "YYYY" """ - from typing import Dict import owid.catalog.processing as pr diff --git a/etl/steps/data/meadow/neglected_tropical_diseases/2024-05-02/soil_transmitted_helminthiases.py b/etl/steps/data/meadow/neglected_tropical_diseases/2024-05-02/soil_transmitted_helminthiases.py index 1906d22ab59..64ff794a725 100644 --- a/etl/steps/data/meadow/neglected_tropical_diseases/2024-05-02/soil_transmitted_helminthiases.py +++ b/etl/steps/data/meadow/neglected_tropical_diseases/2024-05-02/soil_transmitted_helminthiases.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - import numpy as np from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/meadow/ophi/2022-12-13/multidimensional_poverty_index.py b/etl/steps/data/meadow/ophi/2022-12-13/multidimensional_poverty_index.py index 6f7ebee5578..d3b2ad45c11 100644 --- a/etl/steps/data/meadow/ophi/2022-12-13/multidimensional_poverty_index.py +++ b/etl/steps/data/meadow/ophi/2022-12-13/multidimensional_poverty_index.py @@ -1,5 +1,4 @@ -"""Load snapshot of Multidimensional Poverty Index data and prepare a table with basic metadata. -""" +"""Load snapshot of Multidimensional Poverty Index data and prepare a table with basic metadata.""" import pandas as pd from owid.catalog import Dataset, Table, TableMeta diff --git a/etl/steps/data/meadow/papers/2023-02-03/riley_2005/__init__.py b/etl/steps/data/meadow/papers/2023-02-03/riley_2005/__init__.py index aa967c89c90..56238b891d4 100644 --- a/etl/steps/data/meadow/papers/2023-02-03/riley_2005/__init__.py +++ b/etl/steps/data/meadow/papers/2023-02-03/riley_2005/__init__.py @@ -9,6 +9,7 @@ I decided to manually extract the data from the PDF and save it as a CSV and get feedback from authors. """ + import hashlib import pandas as pd diff --git a/etl/steps/data/meadow/papers/2023-07-10/farmer_lafond_2016.py b/etl/steps/data/meadow/papers/2023-07-10/farmer_lafond_2016.py index 64714f53157..f5dd7cba6c5 100644 --- a/etl/steps/data/meadow/papers/2023-07-10/farmer_lafond_2016.py +++ b/etl/steps/data/meadow/papers/2023-07-10/farmer_lafond_2016.py @@ -1,6 +1,4 @@ -"""Load snapshot of Farmer & Lafond (2016) data and create a table. - -""" +"""Load snapshot of Farmer & Lafond (2016) data and create a table.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/meadow/papers/2023-07-10/nemet_2009.py b/etl/steps/data/meadow/papers/2023-07-10/nemet_2009.py index 3b8abc45439..94adfdf13c9 100644 --- a/etl/steps/data/meadow/papers/2023-07-10/nemet_2009.py +++ b/etl/steps/data/meadow/papers/2023-07-10/nemet_2009.py @@ -1,6 +1,4 @@ -"""Load snapshot of Nemet (2009) data and create a table. - -""" +"""Load snapshot of Nemet (2009) data and create a table.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/meadow/papers/2023-12-12/farmer_lafond_2016.py b/etl/steps/data/meadow/papers/2023-12-12/farmer_lafond_2016.py index 70493c8fb68..0748d357b2e 100644 --- a/etl/steps/data/meadow/papers/2023-12-12/farmer_lafond_2016.py +++ b/etl/steps/data/meadow/papers/2023-12-12/farmer_lafond_2016.py @@ -1,6 +1,4 @@ -"""Load snapshot of Farmer & Lafond (2016) data and create a table. - -""" +"""Load snapshot of Farmer & Lafond (2016) data and create a table.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/meadow/papers/2023-12-12/nemet_2009.py b/etl/steps/data/meadow/papers/2023-12-12/nemet_2009.py index 1c47526d1e1..eccb0b0b68e 100644 --- a/etl/steps/data/meadow/papers/2023-12-12/nemet_2009.py +++ b/etl/steps/data/meadow/papers/2023-12-12/nemet_2009.py @@ -1,6 +1,4 @@ -"""Load snapshot of Nemet (2009) data and create a table. - -""" +"""Load snapshot of Nemet (2009) data and create a table.""" from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/meadow/survey/2023-08-04/latinobarometro_trust.py b/etl/steps/data/meadow/survey/2023-08-04/latinobarometro_trust.py index d2e05040b67..25f0a374647 100644 --- a/etl/steps/data/meadow/survey/2023-08-04/latinobarometro_trust.py +++ b/etl/steps/data/meadow/survey/2023-08-04/latinobarometro_trust.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/survey/2023-08-07/afrobarometer_trust.py b/etl/steps/data/meadow/survey/2023-08-07/afrobarometer_trust.py index d09af925f1f..7fb073e1408 100644 --- a/etl/steps/data/meadow/survey/2023-08-07/afrobarometer_trust.py +++ b/etl/steps/data/meadow/survey/2023-08-07/afrobarometer_trust.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/temp/latest/step.py b/etl/steps/data/meadow/temp/latest/step.py index a77f94f00c6..e6a84f4c918 100644 --- a/etl/steps/data/meadow/temp/latest/step.py +++ b/etl/steps/data/meadow/temp/latest/step.py @@ -1,4 +1,5 @@ """Dummy file created for consistency.""" + from etl.helpers import create_dataset diff --git a/etl/steps/data/meadow/un/2023-08-02/comtrade_pandemics.py b/etl/steps/data/meadow/un/2023-08-02/comtrade_pandemics.py index c0553dddb86..67958d66d68 100644 --- a/etl/steps/data/meadow/un/2023-08-02/comtrade_pandemics.py +++ b/etl/steps/data/meadow/un/2023-08-02/comtrade_pandemics.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - import owid.catalog.processing as pr from etl.helpers import PathFinder, create_dataset diff --git a/etl/steps/data/meadow/un/2024-07-11/un_wpp.py b/etl/steps/data/meadow/un/2024-07-11/un_wpp.py index 7a735606361..2be7dd0737e 100644 --- a/etl/steps/data/meadow/un/2024-07-11/un_wpp.py +++ b/etl/steps/data/meadow/un/2024-07-11/un_wpp.py @@ -7,6 +7,7 @@ - Format the tables to have them in long format - Set indices and verify integrity """ + import owid.catalog.processing as pr from owid.catalog import Table from owid.catalog.tables import concat diff --git a/etl/steps/data/meadow/un/2024-07-12/un_wpp.py b/etl/steps/data/meadow/un/2024-07-12/un_wpp.py index 1374b39e23a..547f11cb86b 100644 --- a/etl/steps/data/meadow/un/2024-07-12/un_wpp.py +++ b/etl/steps/data/meadow/un/2024-07-12/un_wpp.py @@ -7,6 +7,7 @@ - Format the tables to have them in long format - Set indices and verify integrity """ + from typing import Dict, List, Optional, Tuple import numpy as np diff --git a/etl/steps/data/meadow/un/2024-07-16/migrant_stock.py b/etl/steps/data/meadow/un/2024-07-16/migrant_stock.py index 24dd7ea5ab3..bf66c27ef2c 100644 --- a/etl/steps/data/meadow/un/2024-07-16/migrant_stock.py +++ b/etl/steps/data/meadow/un/2024-07-16/migrant_stock.py @@ -1,5 +1,6 @@ """Load a snapshot and create a meadow dataset. -Meadow dataset is already very processed to """ +Meadow dataset is already very processed to""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/un/2024-08-27/un_sdg.py b/etl/steps/data/meadow/un/2024-08-27/un_sdg.py index fa79fcace31..9878b6ba2bf 100644 --- a/etl/steps/data/meadow/un/2024-08-27/un_sdg.py +++ b/etl/steps/data/meadow/un/2024-08-27/un_sdg.py @@ -1,4 +1,5 @@ """Load a snapshot and create a meadow dataset.""" + import re import pandas as pd diff --git a/etl/steps/data/meadow/un/2024-12-02/un_wpp_lt.py b/etl/steps/data/meadow/un/2024-12-02/un_wpp_lt.py new file mode 100644 index 00000000000..9535957eedc --- /dev/null +++ b/etl/steps/data/meadow/un/2024-12-02/un_wpp_lt.py @@ -0,0 +1,67 @@ +"""Load a snapshot and create a meadow dataset.""" + +from structlog import get_logger + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) +# Logger +log = get_logger() +# Column rename +COLUMNS_RENAME = { + "mx": "central_death_rate", + "qx": "probability_of_death", + "px": "probability_of_survival", + "lx": "number_survivors", + "dx": "number_deaths", + "Lx": "number_person_years_lived", + "Sx": "survivorship_ratio", + "Tx": "number_person_years_remaining", + "ex": "life_expectancy", + "ax": "average_survival_length", +} + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + snap_short_names = [ + "un_wpp_lt_all", # ALL + "un_wpp_lt_f", # FEMALE + "un_wpp_lt_m", # MALE + ] + + tables = [] + for snap_short_name in snap_short_names: + # Load data from snapshot. + log.info(f"un_wpp_lt: reading {snap_short_name}") + snap = paths.load_snapshot(f"{snap_short_name}.csv") + tb = snap.read_csv( + dtype={ + "Notes": str, + "ISO3_code": "category", + "ISO2_code": "category", + }, + compression="gzip", + ) + # Rename columns + tb = tb.rename(columns=COLUMNS_RENAME) + # Filter only relevant location types + tb = tb[ + tb["LocTypeName"].isin(["Geographic region", "Income group", "Country/Area", "World", "Development group"]) + ] + # Set index + tb = tb.format(["location", "time", "sex", "agegrp", "loctypename"]) + # Add to tables list + tables.append(tb) + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=tables, check_variables_metadata=True, default_metadata=snap.metadata) # type: ignore + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/un_sdg/2021-09-30/un_sdg.ipynb b/etl/steps/data/meadow/un_sdg/2021-09-30/un_sdg.ipynb index bf07ea15252..5fb39bd469e 100644 --- a/etl/steps/data/meadow/un_sdg/2021-09-30/un_sdg.ipynb +++ b/etl/steps/data/meadow/un_sdg/2021-09-30/un_sdg.ipynb @@ -669,9 +669,7 @@ "metadata": {}, "outputs": [], "source": [ - "series = (\n", - " df.SeriesCode.unique()\n", - ") # I think it makes sense to have one table per series code and then drop out the empty columns in each table *but* it would be good to retain a link to the indicators - maybe another table...\n", + "series = df.SeriesCode.unique() # I think it makes sense to have one table per series code and then drop out the empty columns in each table *but* it would be good to retain a link to the indicators - maybe another table...\n", "\n", "df_dict = {elem: pd.DataFrame for elem in series}" ] diff --git a/etl/steps/data/meadow/unicef/2024-07-30/child_migration.py b/etl/steps/data/meadow/unicef/2024-07-30/child_migration.py index 18cc204c639..9939fdb5b67 100644 --- a/etl/steps/data/meadow/unicef/2024-07-30/child_migration.py +++ b/etl/steps/data/meadow/unicef/2024-07-30/child_migration.py @@ -1,4 +1,5 @@ """Load a snapshot and create a meadow dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/urbanization/2024-12-02/ghsl_urban_centers.py b/etl/steps/data/meadow/urbanization/2024-12-02/ghsl_urban_centers.py new file mode 100644 index 00000000000..ac3575f9683 --- /dev/null +++ b/etl/steps/data/meadow/urbanization/2024-12-02/ghsl_urban_centers.py @@ -0,0 +1,112 @@ +"""Load a snapshot and create a meadow dataset.""" + +import owid.catalog.processing as pr +import pandas as pd + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("ghsl_urban_centers.xlsx") + + # Load data from snapshot. + tb_urban_center_names = snap.read(safe_types=False, sheet_name="General_info") + tb_urban_center_density = snap.read(safe_types=False, sheet_name="Area_km2_time_series") + tb_urban_center_population = snap.read(safe_types=False, sheet_name="POP_time_series") + + # Process data. + # + + # Remove duplicates in the ID sheet - based on the name of the urban center and country + tb_urban_center_names = tb_urban_center_names.drop_duplicates(subset=["Main Name", "GADM_name"]) + + tb_urban_center_names = tb_urban_center_names[ + [ + "ID_MTUC_G0", + "Main Name", + "GADM_name", + "UNSDGRegion", + "CountryCapital", + ] + ] + tb_urban_center_density = tb_urban_center_density.melt( + id_vars=["ID_MTUC_G0"], var_name="year", value_name="urban_area" + ) + tb_urban_center_population = tb_urban_center_population.melt( + id_vars=["ID_MTUC_G0"], var_name="year", value_name="urban_pop" + ) + + # Replace zeros with NaNs in the urban_pop column (when the urban center did not meet the criteria) + tb_urban_center_population["urban_pop"] = tb_urban_center_population["urban_pop"].replace(0, pd.NA) + + # Convert the urban_pop column to a numeric dtype + tb_urban_center_population["urban_pop"] = pd.to_numeric(tb_urban_center_population["urban_pop"], errors="coerce") + + tb = pr.merge( + tb_urban_center_population, + tb_urban_center_density, + on=["ID_MTUC_G0", "year"], + how="outer", + ) + tb["urban_density"] = tb["urban_pop"] / tb["urban_area"] + + tb = pr.merge( + tb, + tb_urban_center_names, + on="ID_MTUC_G0", + how="right", + ) + + tb = tb.rename( + columns={ + "GADM_name": "country", + "Main Name": "urban_center_name", + "UNSDGRegion": "region", + "WBIncome2022": "income_group", + "CountryCapital": "capital", + } + ) + + # Filter the Table where urban_center_name is NaN + tb = tb.dropna(subset=["urban_center_name"]) + + # Population and density of the capital city + tb_capitals = tb[tb["capital"] == 1] + + tb_capitals = tb_capitals.drop(columns=["ID_MTUC_G0", "region", "capital"]) + + # Select the top 100 most populous cities in 2020 + tb_2020 = tb[tb["year"] == 2020] + top_100_pop_2020 = tb_2020.nlargest(100, "urban_pop").drop_duplicates(subset=["ID_MTUC_G0"]) + + # Filter the original Table to select the top urban centers + tb_top = tb[tb["ID_MTUC_G0"].isin(top_100_pop_2020["ID_MTUC_G0"])] + + tb_top = tb_top.drop(columns=["urban_area", "ID_MTUC_G0", "region", "capital"]) + tb_top = tb_top.rename(columns={"urban_density": "urban_density_top_100", "urban_pop": "urban_pop_top_100"}) + + # Format the country column + tb_top["country"] = tb_top["urban_center_name"] + " (" + tb_top["country"] + ")" + tb_top = tb_top.drop(columns=["urban_center_name"]) + + tb = pr.merge(tb_capitals, tb_top, on=["country", "year"], how="outer") + + for col in ["urban_pop", "urban_density_top_100", "urban_pop_top_100"]: + tb[col].metadata.origins = tb["country"].metadata.origins + + tb = tb.format(["country", "year"]) + + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=snap.metadata) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/usgs/2024-07-15/historical_statistics_for_mineral_and_material_commodities.py b/etl/steps/data/meadow/usgs/2024-07-15/historical_statistics_for_mineral_and_material_commodities.py index ddfb88f2a2b..5febdd0b6f9 100644 --- a/etl/steps/data/meadow/usgs/2024-07-15/historical_statistics_for_mineral_and_material_commodities.py +++ b/etl/steps/data/meadow/usgs/2024-07-15/historical_statistics_for_mineral_and_material_commodities.py @@ -48,6 +48,8 @@ World production data for 1908 to the most recent year represent the quantity of feldspar that was produced annually throughout the world as reported in the MR and the MYB. World production data do not include production data for nepheline syenite.""", "silver": """World Production World production data for 1900 to the most recent year represent the recoverable silver content of precious-metal ores that were extracted from mines throughout the world. World production data were from the MR and the MYB.""", + "kyanite": """World Production +World production data for 1928-60 were from the “World Production” table in the 1960 MYB. World production data for 1961-70 were from the “World Mine Production” table in the CDS. World production data for 1971-2002 were from the MCS. Data for 2003 to 2006 are unpublished revisions made by the USGS kyanite commodity specialist. World production data for 2007 to the most recent year are from the 2010 to most recent MYB.""", } @@ -244,6 +246,23 @@ def combine_data_for_all_commodities( continue df = clean_sheet_data(data=data, commodity=commodity, sheet_name=sheet_name) + if sheet_name == "Gemstones": + # Gemstones does not contain a "Production" column, but a "Production value ($)" column. + # For now, for consistency with all other files, ignore this column. + df = df.drop(columns=["Production value ($)"], errors="raise") + + if sheet_name == "Ilmenite and Slag": + # For commodity "titanium_mineralconcentrates", the table contains a column "Unit values", with subcolumns for "Ilmenite", "Slag", and "Weighted average". We could attempt to extract the latter, but for now, simply skip. + continue + + if sheet_name == "Lithium statistics": + # Lithium has three different columns for global production, namely "World production (gross weight)", "World production (lithium content)", and "World production (lithium carbonate equivalent)". + # For consistency with USGS current data, we'll use the "World production (lithium content)" column. + # NOTE: The original "World production (gross weight)" column was already renamed to "World production" in clean_sheet_data(). + df = df.drop( + columns=["World production", "World production (lithium carbonate equivalent)"], errors="raise" + ).rename(columns={"World production (lithium content)": "World production"}, errors="raise") + # Add the dataframe for the current commodity to the combined dataframe. combined = pd.concat([combined, df]) diff --git a/etl/steps/data/meadow/wash/2024-01-06/who.py b/etl/steps/data/meadow/wash/2024-01-06/who.py index 2de9833f5f3..7cad00a4594 100644 --- a/etl/steps/data/meadow/wash/2024-01-06/who.py +++ b/etl/steps/data/meadow/wash/2024-01-06/who.py @@ -1,4 +1,5 @@ """Load a snapshot and create a meadow dataset.""" + import numpy as np from owid.catalog import Table from owid.catalog import processing as pr diff --git a/etl/steps/data/meadow/wash/2024-02-15/bathing_water.py b/etl/steps/data/meadow/wash/2024-02-15/bathing_water.py index cfb2741ec6a..ecd6c8e22cc 100644 --- a/etl/steps/data/meadow/wash/2024-02-15/bathing_water.py +++ b/etl/steps/data/meadow/wash/2024-02-15/bathing_water.py @@ -1,4 +1,5 @@ """Load a snapshot and create a meadow dataset.""" + import zipfile from owid.catalog import processing as pr diff --git a/etl/steps/data/meadow/wb/2022-10-29/wb_gender/__init__.py b/etl/steps/data/meadow/wb/2022-10-29/wb_gender/__init__.py index 38b4fc4d470..b2af5fda508 100644 --- a/etl/steps/data/meadow/wb/2022-10-29/wb_gender/__init__.py +++ b/etl/steps/data/meadow/wb/2022-10-29/wb_gender/__init__.py @@ -1,4 +1,5 @@ """WB Gender Meadow step.""" + import re import tempfile from pathlib import Path diff --git a/etl/steps/data/meadow/wb/2024-12-03/poverty_projections.py b/etl/steps/data/meadow/wb/2024-12-03/poverty_projections.py new file mode 100644 index 00000000000..5855e159ba8 --- /dev/null +++ b/etl/steps/data/meadow/wb/2024-12-03/poverty_projections.py @@ -0,0 +1,60 @@ +"""Load a snapshot and create a meadow dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +# Define files directory +FILES_DIRECTORY = "FR_WLD_2024_198/Reproducibility package/Chapter 1/1-data/raw/forecasts" + +# Define index columns +INDEX_COLUMNS = ["country", "year", "povertyline", "scenario"] + +# Define table parameters +TABLE_PARAMETERS = { + "country": {"file": "FGTcountry_1990_2050_3pr24.dta"}, + "region": {"file": "FGTregion_1990_2050_3pr24.dta"}, + "global": {"file": "FGTglobal_1990_2050_3pr24.dta"}, +} + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("reproducibility_package_poverty_prosperity_planet.zip") + + # Define empty list to store tables. + tables = [] + for table, table_config in TABLE_PARAMETERS.items(): + # Load data from snapshot. + tb = snap.read_in_archive(f"{FILES_DIRECTORY}/{table_config['file']}") + + # + # Process data. + # + # Rename and add columns + if table == "region": + tb = tb.rename(columns={"region_pip": "country"}) + elif table == "global": + tb["country"] = "World" + + # Remove duplicates in the data + tb = tb.drop_duplicates(subset=INDEX_COLUMNS) + + # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. + tb = tb.format(keys=INDEX_COLUMNS, short_name=table) + + # Append table to list. + tables.append(tb) + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=tables, check_variables_metadata=True, default_metadata=snap.metadata) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/who/2023-03-09/gho_suicides.py b/etl/steps/data/meadow/who/2023-03-09/gho_suicides.py index 28dc5359eca..e1b8f381e1b 100644 --- a/etl/steps/data/meadow/who/2023-03-09/gho_suicides.py +++ b/etl/steps/data/meadow/who/2023-03-09/gho_suicides.py @@ -1,4 +1,5 @@ """Load a snapshot and create a meadow dataset.""" + import json import pandas as pd diff --git a/etl/steps/data/meadow/who/2023-10-13/medical_devices_atlas.py b/etl/steps/data/meadow/who/2023-10-13/medical_devices_atlas.py index 7ea5eb28c94..0be85a0b562 100644 --- a/etl/steps/data/meadow/who/2023-10-13/medical_devices_atlas.py +++ b/etl/steps/data/meadow/who/2023-10-13/medical_devices_atlas.py @@ -24,9 +24,9 @@ def run(dest_dir: str) -> None: tb = tb[["Indicator", "Location", "Period", "Value"]] tb = tb.rename(columns={"Location": "country", "Period": "year", "Value": "value"}) - tb.loc[ - tb["year"] == "2017-2019", "year" - ] = "2018" # 2017-2019 is the average of 2017, 2018 and 2019 so we replace it with 2018 to be consistent with the other years + tb.loc[tb["year"] == "2017-2019", "year"] = ( + "2018" # 2017-2019 is the average of 2017, 2018 and 2019 so we replace it with 2018 to be consistent with the other years + ) # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. tb = tb.underscore().set_index(["country", "year", "indicator"], verify_integrity=True).sort_index() diff --git a/etl/steps/data/meadow/who/2023-11-01/who_statins.py b/etl/steps/data/meadow/who/2023-11-01/who_statins.py index a6a1b4c9da3..8fc29e89217 100644 --- a/etl/steps/data/meadow/who/2023-11-01/who_statins.py +++ b/etl/steps/data/meadow/who/2023-11-01/who_statins.py @@ -1,4 +1,5 @@ """Load a snapshot and create a meadow dataset.""" + from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/who/2024-03-24/self_inflicted_injuries.py b/etl/steps/data/meadow/who/2024-03-24/self_inflicted_injuries.py index 2324e5df760..f75e75174d3 100644 --- a/etl/steps/data/meadow/who/2024-03-24/self_inflicted_injuries.py +++ b/etl/steps/data/meadow/who/2024-03-24/self_inflicted_injuries.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/who/2024-04-26/avian_influenza_ah5n1.py b/etl/steps/data/meadow/who/2024-04-26/avian_influenza_ah5n1.py index a4bb6343b25..14779bbd97f 100644 --- a/etl/steps/data/meadow/who/2024-04-26/avian_influenza_ah5n1.py +++ b/etl/steps/data/meadow/who/2024-04-26/avian_influenza_ah5n1.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/who/latest/avian_influenza_ah5n1.py b/etl/steps/data/meadow/who/latest/avian_influenza_ah5n1.py index a4bb6343b25..14779bbd97f 100644 --- a/etl/steps/data/meadow/who/latest/avian_influenza_ah5n1.py +++ b/etl/steps/data/meadow/who/latest/avian_influenza_ah5n1.py @@ -1,6 +1,5 @@ """Load a snapshot and create a meadow dataset.""" - from etl.helpers import PathFinder, create_dataset # Get paths and naming conventions for current step. diff --git a/etl/steps/data/meadow/who/latest/flunet.py b/etl/steps/data/meadow/who/latest/flunet.py index c2c5506ae65..b8ad6a7a779 100644 --- a/etl/steps/data/meadow/who/latest/flunet.py +++ b/etl/steps/data/meadow/who/latest/flunet.py @@ -34,7 +34,7 @@ def run(dest_dir: str) -> None: # Convert messy columns to string. # for col in ("aother_subtype_details", "parainfluenza", "otherrespvirus"): - for col in ("aother_subtype_details",): + for col in ("aother_subtype_details", "other_respvirus_details"): ix = tb[col].notnull() tb.loc[ix, col] = tb.loc[ix, col].astype("str") diff --git a/etl/steps/data/meadow/wid/2024-05-24/world_inequality_database.py b/etl/steps/data/meadow/wid/2024-05-24/world_inequality_database.py index 2778c1d89dc..19cb53daa41 100644 --- a/etl/steps/data/meadow/wid/2024-05-24/world_inequality_database.py +++ b/etl/steps/data/meadow/wid/2024-05-24/world_inequality_database.py @@ -1,6 +1,5 @@ """Load a snapshot and create the World Inequality Dataset meadow dataset.""" - import owid.catalog.processing as pr from owid.catalog import Table diff --git a/etl/steps/export/explorers/migration/2024-08-05/migration.py b/etl/steps/export/explorers/migration/2024-08-05/migration.py index 1a3e2c41557..d6bc9c4cf72 100644 --- a/etl/steps/export/explorers/migration/2024-08-05/migration.py +++ b/etl/steps/export/explorers/migration/2024-08-05/migration.py @@ -1,4 +1,5 @@ """Load a grapher dataset and create an explorer dataset with its tsv file.""" + from pathlib import Path import pandas as pd diff --git a/etl/steps/export/explorers/minerals/latest/minerals.py b/etl/steps/export/explorers/minerals/latest/minerals.py index 89b20cee982..436eeed3885 100644 --- a/etl/steps/export/explorers/minerals/latest/minerals.py +++ b/etl/steps/export/explorers/minerals/latest/minerals.py @@ -3,7 +3,7 @@ import pandas as pd from structlog import get_logger -from etl.helpers import PathFinder, create_explorer_old +from etl.helpers import PathFinder, create_explorer # Initialize log. log = get_logger() @@ -190,5 +190,5 @@ def run(dest_dir: str) -> None: # Save outputs. # # Create a new explorers dataset and tsv file. - ds_explorer = create_explorer_old(dest_dir=dest_dir, config=config, df_graphers=df_graphers) + ds_explorer = create_explorer(dest_dir=dest_dir, config=config, df_graphers=df_graphers) ds_explorer.save() diff --git a/etl/steps/export/github/co2_data/latest/owid_co2.py b/etl/steps/export/github/co2_data/latest/owid_co2.py index e1ef156d834..27f669b95e4 100644 --- a/etl/steps/export/github/co2_data/latest/owid_co2.py +++ b/etl/steps/export/github/co2_data/latest/owid_co2.py @@ -14,16 +14,19 @@ * The README file. """ + import os import tempfile from pathlib import Path +import git import pandas as pd from owid.catalog import Table from structlog import get_logger from apps.owidbot import github_utils as gh from etl.helpers import PathFinder +from etl.paths import BASE_DIR # Initialize logger. log = get_logger() @@ -211,13 +214,15 @@ def run(dest_dir: str) -> None: # # Save outputs. # - # If you want to really commit the data, use `CO2_BRANCH=my-branch etlr github/co2_data --export` - if os.environ.get("CO2_BRANCH"): - dry_run = False - branch = os.environ["CO2_BRANCH"] - else: + branch = git.Repo(BASE_DIR).active_branch.name + + if branch == "master": + log.warning("You are on master branch, using dry mode.") dry_run = True - branch = "master" + else: + log.info(f"Committing files to branch {branch}") + # Load DRY_RUN from env or use False as default. + dry_run = bool(int(os.environ.get("DRY_RUN", 0))) # Uncomment to inspect changes. # from etl.data_helpers.misc import compare_tables @@ -233,6 +238,8 @@ def run(dest_dir: str) -> None: prepare_and_save_outputs(tb, codebook=codebook, temp_dir_path=temp_dir_path) + gh.create_branch_if_not_exists(repo_name="co2-data", branch=branch, dry_run=dry_run) + # Commit csv files to the repos. for file_name in ["owid-co2-data.csv", "owid-co2-codebook.csv", "README.md"]: with (temp_dir_path / file_name).open("r") as file_content: @@ -244,3 +251,8 @@ def run(dest_dir: str) -> None: branch=branch, dry_run=dry_run, ) + + if not dry_run: + log.info( + f"Files committed successfully to branch {branch}. Create a PR here https://github.com/owid/co2-data/compare/master...{branch}." + ) diff --git a/etl/steps/export/github/who/latest/monkeypox.py b/etl/steps/export/github/who/latest/monkeypox.py index f26d8e8868f..4f11479c212 100644 --- a/etl/steps/export/github/who/latest/monkeypox.py +++ b/etl/steps/export/github/who/latest/monkeypox.py @@ -1,4 +1,5 @@ """Load a grapher dataset and create an explorer dataset with its tsv file.""" + from apps.owidbot import github_utils as gh from etl import config from etl.helpers import PathFinder diff --git a/etl/steps/export/multidim/energy/latest/energy_prices.py b/etl/steps/export/multidim/energy/latest/energy_prices.py new file mode 100644 index 00000000000..5a9a6e59e08 --- /dev/null +++ b/etl/steps/export/multidim/energy/latest/energy_prices.py @@ -0,0 +1,47 @@ +from etl import multidim +from etl.db import get_engine +from etl.helpers import PathFinder + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load Eurostat data on gas and electricity prices. + ds_grapher = paths.load_dataset("energy_prices") + + # Read table of prices in euros. + tb_annual = ds_grapher.read("energy_prices_annual") + tb_monthly = ds_grapher.read("energy_prices_monthly") + + # + # Process data. + # + # Load configuration from adjacent yaml file. + config = paths.load_mdim_config() + + # Create views. + config["views"] = multidim.generate_views_for_dimensions( + dimensions=config["dimensions"], + tables=[tb_annual, tb_monthly], + dimensions_order_in_slug=("frequency", "source", "consumer", "price_component", "unit"), + warn_on_missing_combinations=False, + additional_config={ + "$schema": "https://files.ourworldindata.org/schemas/grapher-schema.005.json", + "chartTypes": ["LineChart"], + "hasMapTab": True, + "tab": "map", + "map": { + "projection": "Europe", + "colorScale": {"baseColorScheme": "YlOrBr"}, + }, + }, + ) + + # + # Save outputs. + # + multidim.upsert_multidim_data_page(slug="mdd-energy-prices", config=config, engine=get_engine()) diff --git a/etl/steps/export/multidim/energy/latest/energy_prices.yml b/etl/steps/export/multidim/energy/latest/energy_prices.yml new file mode 100644 index 00000000000..f2640249e48 --- /dev/null +++ b/etl/steps/export/multidim/energy/latest/energy_prices.yml @@ -0,0 +1,102 @@ +title: + title: "Energy prices" + titleVariant: "by energy source" +defaultSelection: + - "European Union (27)" +topicTags: + - "Energy" +dimensions: + - slug: "frequency" + name: "Frequency" + choices: + - slug: "annual" + name: "Annual" + description: "Annual data" + - slug: "monthly" + name: "Monthly" + description: "Monthly data" + - slug: "source" + name: "Energy source" + choices: + - slug: "electricity" + name: "Electricity" + - slug: "gas" + name: "Gas" + - slug: "price_component" + name: "Price component" + choices: + - slug: "total_price_including_taxes" + name: "Total consumer price" + description: "Total consumer price including all taxes and levies" + group: Overview + - slug: "wholesale" + name: "Wholesale price" + description: "Wholesale price" + group: Overview + - slug: "energy_and_supply" + name: "Energy and supply" + description: "Energy and supply" + group: Individual price components + - slug: "network_costs" + name: "Network costs" + description: "Network costs" + group: Individual price components + - slug: "capacity_taxes" + name: "Capacity taxes" + description: "Capacity taxes" + group: Individual price components + - slug: "value_added_tax_vat" + name: "Value added tax (VAT)" + description: "Value added tax (VAT)" + group: Individual price components + - slug: "environmental_taxes" + name: "Environmental taxes" + description: "Environmental taxes" + group: Individual price components + - slug: "nuclear_taxes" + name: "Nuclear taxes" + description: "Nuclear taxes" + group: Individual price components + - slug: "renewable_taxes" + name: "Renewable taxes" + description: "Renewable taxes" + group: Individual price components + - slug: "taxes_fees_levies_and_charges" + name: "All taxes, fees, levies and charges" + description: "All taxes, fees, levies and charges" + group: Individual price components + - slug: "other" + name: "Other costs" + description: "Other costs" + group: Individual price components + # Other available price components: + # 'capacity_taxes_allowances', + # 'environmental_taxes_allowance', + # 'nuclear_taxes_allowance', + # 'renewable_taxes_allowance', + # 'taxes_fees_levies_and_charges_allowance', + # 'other_allowance', + - slug: "consumer" + name: "Consumer type" + choices: + - slug: "household" + name: "Households" + description: "Household consumers" + - slug: "non_household" + name: "Non-households" + description: "Non-household consumers" + - slug: "all" + name: "All consumers" + description: "All consumers" + - slug: "unit" + name: "Unit" + choices: + - slug: "euro" + name: "Euro" + description: "Price in euros" + - slug: "pps" + name: "PPS" + description: "Price in Purchasing Power Standard" +views: + # Views will be filled out programmatically. + [] diff --git a/etl/steps/export/s3/co2_data/latest/owid_co2.py b/etl/steps/export/s3/co2_data/latest/owid_co2.py index 65973768684..2a66c829220 100644 --- a/etl/steps/export/s3/co2_data/latest/owid_co2.py +++ b/etl/steps/export/s3/co2_data/latest/owid_co2.py @@ -16,6 +16,7 @@ * https://nyc3.digitaloceanspaces.com/owid-public/data/co2/owid-co2-data.json """ + import json import tempfile from pathlib import Path diff --git a/etl/steps/open_numbers.py b/etl/steps/open_numbers.py index 607bda3a3de..f490f406044 100644 --- a/etl/steps/open_numbers.py +++ b/etl/steps/open_numbers.py @@ -13,7 +13,7 @@ import hashlib import tempfile import warnings -from multiprocessing import Pool +from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import Dict, List, Tuple, cast @@ -58,9 +58,9 @@ def run(dest_dir: str) -> None: resource_map = remap_names(package.resources) # copy tables one by one - with Pool() as pool: + with ThreadPoolExecutor() as executor: args = [(ds, repo, short_name, resources) for short_name, resources in resource_map.items()] - pool.starmap(add_resource, args) + executor.map(lambda p: add_resource(*p), args) def add_resource( diff --git a/etl/version_tracker.py b/etl/version_tracker.py index 57f73541748..f75f9f8bbc0 100644 --- a/etl/version_tracker.py +++ b/etl/version_tracker.py @@ -446,53 +446,41 @@ def get_path_to_script(self, step: str, omit_base_dir: bool = False) -> Optional """Get the path to the script of a given step.""" # Get step attributes. _, step_type, _, channel, namespace, version, name, _ = extract_step_attributes(step=step).values() - state = "active" if step in self.all_active_steps else "archive" # Create a dictionary that contains the path to a script for a given step. # This dictionary has to keys, namely "active" and "archive". # Active steps should have a script in the active directory. # But steps that are in the archive dag can be either in the active or the archive directory. - path_to_script = {"active": None, "archive": None} + path_to_script = None if step_type == "export": - path_to_script["active"] = paths.STEP_DIR / "export" / channel / namespace / version / name # type: ignore + path_to_script = paths.STEP_DIR / "export" / channel / namespace / version / name # type: ignore elif channel == "snapshot": - path_to_script["active"] = paths.SNAPSHOTS_DIR / namespace / version / name # type: ignore - path_to_script["archive"] = paths.SNAPSHOTS_DIR_ARCHIVE / namespace / version / name # type: ignore + path_to_script = paths.SNAPSHOTS_DIR / namespace / version / name # type: ignore elif channel in ["meadow", "garden", "grapher", "explorers", "open_numbers", "examples", "external"]: - path_to_script["active"] = paths.STEP_DIR / "data" / channel / namespace / version / name # type: ignore - path_to_script["archive"] = paths.STEP_DIR_ARCHIVE / channel / namespace / version / name # type: ignore + path_to_script = paths.STEP_DIR / "data" / channel / namespace / version / name # type: ignore elif channel == "walden": - path_to_script["active"] = paths.BASE_DIR / "lib" / "walden" / "ingests" / namespace / version / name # type: ignore - path_to_script["archive"] = paths.BASE_DIR / "lib" / "walden" / "ingests" / namespace / version / name # type: ignore + path_to_script = paths.BASE_DIR / "lib" / "walden" / "ingests" / namespace / version / name # type: ignore elif channel in ["backport", "etag"]: # Ignore these channels, for which there is never a script. return None else: log.error(f"Unknown channel {channel} for step {step}.") - if state == "active": - # Steps in the active dag should only have a script in the active directory. - del path_to_script["archive"] - path_to_script_detected = None - for state in path_to_script: - # A step script can exist either as a .py file, as a .ipynb file, or a __init__.py file inside a folder. - # In the case of snapshots, there may or may not be a .py file, but there definitely needs to be a dvc file. - # In that case, the corresponding script is not trivial to find, but at least we can return the dvc file. - for path_to_script_candidate in [ - path_to_script[state].with_suffix(".py"), # type: ignore - path_to_script[state].with_suffix(".ipynb"), # type: ignore - path_to_script[state] / "__init__.py", # type: ignore - path_to_script[state].with_name(path_to_script[state].name + ".dvc"), # type: ignore - ]: - if path_to_script_candidate.exists(): - path_to_script_detected = path_to_script_candidate - break + # A step script can exist either as a .py file, as a .ipynb file, or a __init__.py file inside a folder. + # In the case of snapshots, there may or may not be a .py file, but there definitely needs to be a dvc file. + # In that case, the corresponding script is not trivial to find, but at least we can return the dvc file. + for path_to_script_candidate in [ + path_to_script.with_suffix(".py"), # type: ignore + path_to_script.with_suffix(".ipynb"), # type: ignore + path_to_script / "__init__.py", # type: ignore + path_to_script.with_name(path_to_script.name + ".dvc"), # type: ignore + ]: + if path_to_script_candidate.exists(): + path_to_script_detected = path_to_script_candidate + break if path_to_script_detected is None: - if state == "active": - log.error(f"Script for step {step} not found.") - else: - log.warning(f"Script for archive step {step} not found.") + log.error(f"Script for step {step} not found.") if omit_base_dir and path_to_script_detected is not None: # Return the path relative to the base directory (omitting the local path to the ETL repos). @@ -633,14 +621,14 @@ def _days_since_step_creation(version): ] = UpdateState.ARCHIVABLE.value # There are special steps that, even though they are archivable or unused, we want to keep in the active dag. - steps_active_df.loc[ - steps_active_df["step"].isin(self.ARCHIVABLE_STEPS_TO_KEEP), "update_state" - ] = UpdateState.UP_TO_DATE.value + steps_active_df.loc[steps_active_df["step"].isin(self.ARCHIVABLE_STEPS_TO_KEEP), "update_state"] = ( + UpdateState.UP_TO_DATE.value + ) # All explorers and external steps should be considered up to date. - steps_active_df.loc[ - steps_active_df["channel"].isin(["explorers", "external"]), "update_state" - ] = UpdateState.UP_TO_DATE.value + steps_active_df.loc[steps_active_df["channel"].isin(["explorers", "external"]), "update_state"] = ( + UpdateState.UP_TO_DATE.value + ) # Add update state to archived steps. steps_inactive_df["update_state"] = UpdateState.ARCHIVED.value diff --git a/lib/catalog/owid/catalog/datasets.py b/lib/catalog/owid/catalog/datasets.py index 745563a2d81..938ec6e6480 100644 --- a/lib/catalog/owid/catalog/datasets.py +++ b/lib/catalog/owid/catalog/datasets.py @@ -6,6 +6,7 @@ import json import shutil import warnings +from _hashlib import HASH from dataclasses import dataclass from glob import glob from os import environ @@ -16,7 +17,6 @@ import numpy as np import pandas as pd import yaml -from _hashlib import HASH from owid.repack import to_safe_types @@ -119,7 +119,7 @@ def add( utils.validate_underscore(col, "Variable's name") if not table.primary_key: - if "OWID_STRICT" in environ: + if environ.get("OWID_STRICT"): raise PrimaryKeyMissing( f"Table `{table.metadata.short_name}` does not have a primary_key -- please use t.set_index([col, ...], verify_integrity=True) to indicate dimensions before saving" ) @@ -128,7 +128,7 @@ def add( f"Table `{table.metadata.short_name}` does not have a primary_key -- please use t.set_index([col, ...], verify_integrity=True) to indicate dimensions before saving" ) - if not table.index.is_unique and "OWID_STRICT" in environ: + if not table.index.is_unique and environ.get("OWID_STRICT"): [(k, dups)] = table.index.value_counts().head(1).to_dict().items() raise NonUniqueIndex( f"Table `{table.metadata.short_name}` has duplicate values in the index -- could you have made a mistake?\n\n" diff --git a/lib/catalog/owid/catalog/processing.py b/lib/catalog/owid/catalog/processing.py index 741ecbb383e..0eb62128620 100755 --- a/lib/catalog/owid/catalog/processing.py +++ b/lib/catalog/owid/catalog/processing.py @@ -1,6 +1,5 @@ -"""Common operations performed on tables and variables. +"""Common operations performed on tables and variables.""" -""" from .tables import ( ExcelFile, concat, @@ -20,6 +19,7 @@ read_json, read_parquet, read_rda, + read_rda_multiple, read_rds, read_stata, to_numeric, @@ -42,6 +42,7 @@ "read_fwf", "read_stata", "read_rda", + "read_rda_multiple", "read_rds", "read_df", "read_parquet", diff --git a/lib/catalog/owid/catalog/processing_log.py b/lib/catalog/owid/catalog/processing_log.py index 389c4fe7e0e..2581285b402 100644 --- a/lib/catalog/owid/catalog/processing_log.py +++ b/lib/catalog/owid/catalog/processing_log.py @@ -50,12 +50,10 @@ class LogEntry: parents: Tuple[str, ...] = field(default_factory=tuple) comment: Optional[str] = None - def to_dict(self) -> Dict[str, Any]: - ... + def to_dict(self) -> Dict[str, Any]: ... @staticmethod - def from_dict(d: Dict[str, Any]) -> "LogEntry": - ... + def from_dict(d: Dict[str, Any]) -> "LogEntry": ... def clone(self, **kwargs): """Clone the log entry, optionally overriding some attributes.""" diff --git a/lib/catalog/owid/catalog/tables.py b/lib/catalog/owid/catalog/tables.py index 88fa0e1f9e2..6fb0978e9d0 100644 --- a/lib/catalog/owid/catalog/tables.py +++ b/lib/catalog/owid/catalog/tables.py @@ -192,12 +192,10 @@ def read(cls, path: Union[str, Path], **kwargs) -> "Table": return table @overload - def to_csv(self, path: None = None, **kwargs: Any) -> str: - ... + def to_csv(self, path: None = None, **kwargs: Any) -> str: ... @overload - def to_csv(self, path: Any, **kwargs: Any) -> None: - ... + def to_csv(self, path: Any, **kwargs: Any) -> None: ... def to_csv(self, path: Optional[Any] = None, **kwargs: Any) -> Union[None, str]: """ @@ -445,16 +443,13 @@ def rename( *, inplace: Literal[True], **kwargs: Any, - ) -> None: - ... + ) -> None: ... @overload - def rename(self, mapper: Any = None, *, inplace: Literal[False], **kwargs: Any) -> "Table": - ... + def rename(self, mapper: Any = None, *, inplace: Literal[False], **kwargs: Any) -> "Table": ... @overload - def rename(self, *args: Any, **kwargs: Any) -> "Table": - ... + def rename(self, *args: Any, **kwargs: Any) -> "Table": ... def rename(self, *args: Any, **kwargs: Any) -> Optional["Table"]: """Rename columns while keeping their metadata.""" @@ -559,16 +554,13 @@ def set_index( *, inplace: Literal[True], **kwargs: Any, - ) -> None: - ... + ) -> None: ... @overload - def set_index(self, keys: Union[str, List[str]], *, inplace: Literal[False], **kwargs: Any) -> "Table": - ... + def set_index(self, keys: Union[str, List[str]], *, inplace: Literal[False], **kwargs: Any) -> "Table": ... @overload - def set_index(self, keys: Union[str, List[str]], **kwargs: Any) -> "Table": - ... + def set_index(self, keys: Union[str, List[str]], **kwargs: Any) -> "Table": ... def set_index( self, @@ -588,16 +580,13 @@ def set_index( return cast(Table, t) @overload - def reset_index(self, level=None, *, inplace: Literal[True], **kwargs) -> None: - ... + def reset_index(self, level=None, *, inplace: Literal[True], **kwargs) -> None: ... @overload - def reset_index(self, level=None, *, inplace: Literal[False], **kwargs) -> "Table": - ... + def reset_index(self, level=None, *, inplace: Literal[False], **kwargs) -> "Table": ... @overload - def reset_index(self, level=None, *, inplace: bool = False, **kwargs) -> "Table": - ... + def reset_index(self, level=None, *, inplace: bool = False, **kwargs) -> "Table": ... def reset_index(self, level=None, *, inplace: bool = False, **kwargs) -> Optional["Table"]: # type: ignore """Fix type signature of reset_index.""" @@ -617,16 +606,13 @@ def reindex(self, *args, **kwargs) -> "Table": return cast(Table, t) @overload - def drop_duplicates(self, *, inplace: Literal[True], **kwargs) -> None: - ... + def drop_duplicates(self, *, inplace: Literal[True], **kwargs) -> None: ... @overload - def drop_duplicates(self, *, inplace: Literal[False], **kwargs) -> "Table": - ... + def drop_duplicates(self, *, inplace: Literal[False], **kwargs) -> "Table": ... @overload - def drop_duplicates(self, **kwargs) -> "Table": - ... + def drop_duplicates(self, **kwargs) -> "Table": ... def drop_duplicates(self, *args, **kwargs) -> Optional["Table"]: return super().drop_duplicates(*args, **kwargs) @@ -817,16 +803,13 @@ def format( return t @overload - def dropna(self, *, inplace: Literal[True], **kwargs) -> None: - ... + def dropna(self, *, inplace: Literal[True], **kwargs) -> None: ... @overload - def dropna(self, *, inplace: Literal[False], **kwargs) -> "Table": - ... + def dropna(self, *, inplace: Literal[False], **kwargs) -> "Table": ... @overload - def dropna(self, **kwargs) -> "Table": - ... + def dropna(self, **kwargs) -> "Table": ... def dropna(self, *args, **kwargs) -> Optional["Table"]: tb = super().dropna(*args, **kwargs) @@ -1036,7 +1019,7 @@ def fillna(self, value=None, **kwargs) -> "Table": if value is not None: tb = super().fillna(value, **kwargs) - if type(value) == type(self): + if type(value) is type(self): for column in tb.columns: if column in value.columns: tb._fields[column] = variables.combine_variables_metadata( @@ -1072,12 +1055,10 @@ def __init__(self, groupby: pd.core.groupby.DataFrameGroupBy, metadata: TableMet self._fields = fields @overload - def __getattr__(self, name: Literal["count", "size", "sum", "mean", "median"]) -> Callable[[], "Table"]: - ... + def __getattr__(self, name: Literal["count", "size", "sum", "mean", "median"]) -> Callable[[], "Table"]: ... @overload - def __getattr__(self, name: str) -> "VariableGroupBy": - ... + def __getattr__(self, name: str) -> "VariableGroupBy": ... def __getattr__(self, name: str) -> Union[Callable[..., "Table"], "VariableGroupBy"]: # Calling method on the groupby object @@ -1108,12 +1089,10 @@ def func(*args, **kwargs): return VariableGroupBy(getattr(self.groupby, name), name, self._fields[name], self.metadata) @overload - def __getitem__(self, key: str) -> "VariableGroupBy": - ... + def __getitem__(self, key: str) -> "VariableGroupBy": ... @overload - def __getitem__(self, key: list) -> "TableGroupBy": - ... + def __getitem__(self, key: list) -> "TableGroupBy": ... def __getitem__(self, key: Union[str, list]) -> Union["VariableGroupBy", "TableGroupBy"]: if isinstance(key, list): @@ -1159,9 +1138,9 @@ def f(g): df = self.groupby.apply(f, *args, include_groups=include_groups) if not mem: return Table(df) - elif type(mem["table"]) == pd.DataFrame: + elif type(mem["table"]) is pd.DataFrame: return _create_table(df, self.metadata, self._fields) - elif type(mem["table"]) == pd.Series: + elif type(mem["table"]) is pd.Series: if isinstance(df, Table): return _create_table(df, self.metadata, self._fields) else: @@ -1770,6 +1749,51 @@ def read_rda( return cast(Table, table) +def read_rda_multiple( + filepath_or_buffer: Union[str, Path, IO[AnyStr]], + table_names: Optional[List[str]] = None, + metadata: Optional[TableMeta] = None, + origin: Optional[Origin] = None, + underscore: bool = False, +) -> Dict[str, Table]: + # Read RData + parsed = rdata.parser.parse_file(filepath_or_buffer) # type: ignore + converted = rdata.conversion.convert(parsed) + + # Init output dictionary + tables = {} + + if table_names is not None: + # If table names are given, read them + for tname in table_names: + # Check that name exists in RDA file + if tname not in converted: + raise ValueError(f"Table {tname} not found in RDA file.") + # Load object + table = converted[tname] + # Check that object is a DataFrame (otherwise raise error!). NOTE: here we raise an error, bc user explicitly asked us to load this table. + if isinstance(table, pd.DataFrame): + raise ValueError(f"Table {tname} is not a DataFrame.") + # Parse object to Table, and add metadata + table = Table(converted[tname], underscore=underscore) + table = _add_table_and_variables_metadata_to_table(table=table, metadata=metadata, origin=origin) + table.metadata.short_name = tname + # Safe table in main dictionary object + tables[tname] = cast(Table, table) + else: + # Read them all (only those objects that are Dataframes) + for fname, data in converted.items(): + if isinstance(data, pd.DataFrame): + # Parse object to Table, and add metadata + table = Table(data, underscore=underscore) + table = _add_table_and_variables_metadata_to_table(table=table, metadata=metadata, origin=origin) + table.metadata.short_name = fname + # Safe table in main dictionary object + tables[fname] = cast(Table, table) + + return tables + + def read_rds( filepath_or_buffer: Union[str, Path, IO[AnyStr]], metadata: Optional[TableMeta] = None, diff --git a/lib/catalog/owid/catalog/utils.py b/lib/catalog/owid/catalog/utils.py index bd0fe80dc8a..c0dd0a65c71 100644 --- a/lib/catalog/owid/catalog/utils.py +++ b/lib/catalog/owid/catalog/utils.py @@ -42,13 +42,11 @@ def pruned_json(cls: T) -> T: @overload -def underscore(name: str, validate: bool = True, camel_to_snake: bool = False) -> str: - ... +def underscore(name: str, validate: bool = True, camel_to_snake: bool = False) -> str: ... @overload -def underscore(name: None, validate: bool = True, camel_to_snake: bool = False) -> None: - ... +def underscore(name: None, validate: bool = True, camel_to_snake: bool = False) -> None: ... def underscore(name: Optional[str], validate: bool = True, camel_to_snake: bool = False) -> Optional[str]: diff --git a/lib/catalog/owid/catalog/variables.py b/lib/catalog/owid/catalog/variables.py index f6e529b567d..2d54bfc9e8d 100644 --- a/lib/catalog/owid/catalog/variables.py +++ b/lib/catalog/owid/catalog/variables.py @@ -575,13 +575,11 @@ def combine_variables_metadata( @overload -def copy_metadata(from_variable: Variable, to_variable: Variable, inplace: Literal[False] = False) -> Variable: - ... +def copy_metadata(from_variable: Variable, to_variable: Variable, inplace: Literal[False] = False) -> Variable: ... @overload -def copy_metadata(from_variable: Variable, to_variable: Variable, inplace: Literal[True] = True) -> None: - ... +def copy_metadata(from_variable: Variable, to_variable: Variable, inplace: Literal[True] = True) -> None: ... def copy_metadata(from_variable: Variable, to_variable: Variable, inplace: bool = False) -> Optional[Variable]: diff --git a/lib/catalog/pyproject.toml b/lib/catalog/pyproject.toml index 415ece029fe..984bad9d971 100644 --- a/lib/catalog/pyproject.toml +++ b/lib/catalog/pyproject.toml @@ -33,7 +33,7 @@ dev-dependencies = [ "argh>=0.26.2", "pyright==1.1.373", "pandas-stubs==1.2.0.62", - "ruff==0.1.6" + "ruff==0.8.2" ] [tool.uv.sources] diff --git a/lib/catalog/tests/mocking.py b/lib/catalog/tests/mocking.py index cf42a213e5e..9e0e6c1c954 100644 --- a/lib/catalog/tests/mocking.py +++ b/lib/catalog/tests/mocking.py @@ -40,19 +40,19 @@ def mock(_type: type) -> Any: if hasattr(_type, "__forward_arg__"): raise ValueError(_type) - if _type == int: + if _type is int: return random.randint(0, 1000) - elif _type == bool: + elif _type is bool: return random.choice([True, False]) - elif _type == float: + elif _type is float: return 10 * random.random() / random.random() - elif _type == dt.date: + elif _type is dt.date: return _random_date() - elif _type == str: + elif _type is str: # some strings in the frictionless standard must be lowercase with no spaces return random.choice(_MOCK_STRINGS).lower() @@ -72,7 +72,7 @@ def mock(_type: type) -> Any: elif getattr(_type, "__name__", None) == "ProcessingLog": return _type([]) - elif _type == Any: + elif _type is Any: return mock(random.choice([str, int, float])) elif getattr(_type, "__name__", None) == "YearDateLatest": diff --git a/lib/catalog/tests/test_meta.py b/lib/catalog/tests/test_meta.py index 4dfaac4947a..36724ca556e 100644 --- a/lib/catalog/tests/test_meta.py +++ b/lib/catalog/tests/test_meta.py @@ -20,8 +20,7 @@ class Dog: name: Optional[str] = None age: Optional[int] = None - def to_dict(self) -> Dict[str, Any]: - ... + def to_dict(self) -> Dict[str, Any]: ... assert Dog(name="fred").to_dict() == {"name": "fred"} assert Dog(age=10).to_dict() == {"age": 10} @@ -43,8 +42,7 @@ class Dog: age: Optional[int] = None cat: Optional[Cat] = None - def to_dict(self) -> Dict[str, Any]: - ... + def to_dict(self) -> Dict[str, Any]: ... assert Dog(name="fred", cat=Cat(name="cred")).to_dict() == {"name": "fred", "cat": {"name": "cred"}} diff --git a/lib/catalog/uv.lock b/lib/catalog/uv.lock index 2d2a84f22b5..89dcc92754b 100644 --- a/lib/catalog/uv.lock +++ b/lib/catalog/uv.lock @@ -77,6 +77,119 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/2e/ca478472e3a6cc96a23dcaf82af714e2befbf449aec98974bf0ac2c88102/botocore-1.35.20-py3-none-any.whl", hash = "sha256:62412038f960691a299e60492f9ee7e8e75af563f2eca7f3640b3b54b8f5d236", size = 12539792 }, ] +[[package]] +name = "brotli" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/c2/f9e977608bdf958650638c3f1e28f85a1b075f075ebbe77db8555463787b/Brotli-1.1.0.tar.gz", hash = "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724", size = 7372270 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/3a/dbf4fb970c1019a57b5e492e1e0eae745d32e59ba4d6161ab5422b08eefe/Brotli-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e1140c64812cb9b06c922e77f1c26a75ec5e3f0fb2bf92cc8c58720dec276752", size = 873045 }, + { url = "https://files.pythonhosted.org/packages/dd/11/afc14026ea7f44bd6eb9316d800d439d092c8d508752055ce8d03086079a/Brotli-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c8fd5270e906eef71d4a8d19b7c6a43760c6abcfcc10c9101d14eb2357418de9", size = 446218 }, + { url = "https://files.pythonhosted.org/packages/36/83/7545a6e7729db43cb36c4287ae388d6885c85a86dd251768a47015dfde32/Brotli-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ae56aca0402a0f9a3431cddda62ad71666ca9d4dc3a10a142b9dce2e3c0cda3", size = 2903872 }, + { url = "https://files.pythonhosted.org/packages/32/23/35331c4d9391fcc0f29fd9bec2c76e4b4eeab769afbc4b11dd2e1098fb13/Brotli-1.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:43ce1b9935bfa1ede40028054d7f48b5469cd02733a365eec8a329ffd342915d", size = 2941254 }, + { url = "https://files.pythonhosted.org/packages/3b/24/1671acb450c902edb64bd765d73603797c6c7280a9ada85a195f6b78c6e5/Brotli-1.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:7c4855522edb2e6ae7fdb58e07c3ba9111e7621a8956f481c68d5d979c93032e", size = 2857293 }, + { url = "https://files.pythonhosted.org/packages/d5/00/40f760cc27007912b327fe15bf6bfd8eaecbe451687f72a8abc587d503b3/Brotli-1.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:38025d9f30cf4634f8309c6874ef871b841eb3c347e90b0851f63d1ded5212da", size = 3002385 }, + { url = "https://files.pythonhosted.org/packages/b8/cb/8aaa83f7a4caa131757668c0fb0c4b6384b09ffa77f2fba9570d87ab587d/Brotli-1.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e6a904cb26bfefc2f0a6f240bdf5233be78cd2488900a2f846f3c3ac8489ab80", size = 2911104 }, + { url = "https://files.pythonhosted.org/packages/bc/c4/65456561d89d3c49f46b7fbeb8fe6e449f13bdc8ea7791832c5d476b2faf/Brotli-1.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a37b8f0391212d29b3a91a799c8e4a2855e0576911cdfb2515487e30e322253d", size = 2809981 }, + { url = "https://files.pythonhosted.org/packages/05/1b/cf49528437bae28abce5f6e059f0d0be6fecdcc1d3e33e7c54b3ca498425/Brotli-1.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e84799f09591700a4154154cab9787452925578841a94321d5ee8fb9a9a328f0", size = 2935297 }, + { url = "https://files.pythonhosted.org/packages/81/ff/190d4af610680bf0c5a09eb5d1eac6e99c7c8e216440f9c7cfd42b7adab5/Brotli-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f66b5337fa213f1da0d9000bc8dc0cb5b896b726eefd9c6046f699b169c41b9e", size = 2930735 }, + { url = "https://files.pythonhosted.org/packages/80/7d/f1abbc0c98f6e09abd3cad63ec34af17abc4c44f308a7a539010f79aae7a/Brotli-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5dab0844f2cf82be357a0eb11a9087f70c5430b2c241493fc122bb6f2bb0917c", size = 2933107 }, + { url = "https://files.pythonhosted.org/packages/34/ce/5a5020ba48f2b5a4ad1c0522d095ad5847a0be508e7d7569c8630ce25062/Brotli-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e4fe605b917c70283db7dfe5ada75e04561479075761a0b3866c081d035b01c1", size = 2845400 }, + { url = "https://files.pythonhosted.org/packages/44/89/fa2c4355ab1eecf3994e5a0a7f5492c6ff81dfcb5f9ba7859bd534bb5c1a/Brotli-1.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1e9a65b5736232e7a7f91ff3d02277f11d339bf34099a56cdab6a8b3410a02b2", size = 3031985 }, + { url = "https://files.pythonhosted.org/packages/af/a4/79196b4a1674143d19dca400866b1a4d1a089040df7b93b88ebae81f3447/Brotli-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:58d4b711689366d4a03ac7957ab8c28890415e267f9b6589969e74b6e42225ec", size = 2927099 }, + { url = "https://files.pythonhosted.org/packages/e9/54/1c0278556a097f9651e657b873ab08f01b9a9ae4cac128ceb66427d7cd20/Brotli-1.1.0-cp310-cp310-win32.whl", hash = "sha256:be36e3d172dc816333f33520154d708a2657ea63762ec16b62ece02ab5e4daf2", size = 333172 }, + { url = "https://files.pythonhosted.org/packages/f7/65/b785722e941193fd8b571afd9edbec2a9b838ddec4375d8af33a50b8dab9/Brotli-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:0c6244521dda65ea562d5a69b9a26120769b7a9fb3db2fe9545935ed6735b128", size = 357255 }, + { url = "https://files.pythonhosted.org/packages/96/12/ad41e7fadd5db55459c4c401842b47f7fee51068f86dd2894dd0dcfc2d2a/Brotli-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a3daabb76a78f829cafc365531c972016e4aa8d5b4bf60660ad8ecee19df7ccc", size = 873068 }, + { url = "https://files.pythonhosted.org/packages/95/4e/5afab7b2b4b61a84e9c75b17814198ce515343a44e2ed4488fac314cd0a9/Brotli-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c8146669223164fc87a7e3de9f81e9423c67a79d6b3447994dfb9c95da16e2d6", size = 446244 }, + { url = "https://files.pythonhosted.org/packages/9d/e6/f305eb61fb9a8580c525478a4a34c5ae1a9bcb12c3aee619114940bc513d/Brotli-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30924eb4c57903d5a7526b08ef4a584acc22ab1ffa085faceb521521d2de32dd", size = 2906500 }, + { url = "https://files.pythonhosted.org/packages/3e/4f/af6846cfbc1550a3024e5d3775ede1e00474c40882c7bf5b37a43ca35e91/Brotli-1.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ceb64bbc6eac5a140ca649003756940f8d6a7c444a68af170b3187623b43bebf", size = 2943950 }, + { url = "https://files.pythonhosted.org/packages/b3/e7/ca2993c7682d8629b62630ebf0d1f3bb3d579e667ce8e7ca03a0a0576a2d/Brotli-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a469274ad18dc0e4d316eefa616d1d0c2ff9da369af19fa6f3daa4f09671fd61", size = 2918527 }, + { url = "https://files.pythonhosted.org/packages/b3/96/da98e7bedc4c51104d29cc61e5f449a502dd3dbc211944546a4cc65500d3/Brotli-1.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:524f35912131cc2cabb00edfd8d573b07f2d9f21fa824bd3fb19725a9cf06327", size = 2845489 }, + { url = "https://files.pythonhosted.org/packages/e8/ef/ccbc16947d6ce943a7f57e1a40596c75859eeb6d279c6994eddd69615265/Brotli-1.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5b3cc074004d968722f51e550b41a27be656ec48f8afaeeb45ebf65b561481dd", size = 2914080 }, + { url = "https://files.pythonhosted.org/packages/80/d6/0bd38d758d1afa62a5524172f0b18626bb2392d717ff94806f741fcd5ee9/Brotli-1.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:19c116e796420b0cee3da1ccec3b764ed2952ccfcc298b55a10e5610ad7885f9", size = 2813051 }, + { url = "https://files.pythonhosted.org/packages/14/56/48859dd5d129d7519e001f06dcfbb6e2cf6db92b2702c0c2ce7d97e086c1/Brotli-1.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:510b5b1bfbe20e1a7b3baf5fed9e9451873559a976c1a78eebaa3b86c57b4265", size = 2938172 }, + { url = "https://files.pythonhosted.org/packages/3d/77/a236d5f8cd9e9f4348da5acc75ab032ab1ab2c03cc8f430d24eea2672888/Brotli-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a1fd8a29719ccce974d523580987b7f8229aeace506952fa9ce1d53a033873c8", size = 2933023 }, + { url = "https://files.pythonhosted.org/packages/f1/87/3b283efc0f5cb35f7f84c0c240b1e1a1003a5e47141a4881bf87c86d0ce2/Brotli-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c247dd99d39e0338a604f8c2b3bc7061d5c2e9e2ac7ba9cc1be5a69cb6cd832f", size = 2935871 }, + { url = "https://files.pythonhosted.org/packages/f3/eb/2be4cc3e2141dc1a43ad4ca1875a72088229de38c68e842746b342667b2a/Brotli-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1b2c248cd517c222d89e74669a4adfa5577e06ab68771a529060cf5a156e9757", size = 2847784 }, + { url = "https://files.pythonhosted.org/packages/66/13/b58ddebfd35edde572ccefe6890cf7c493f0c319aad2a5badee134b4d8ec/Brotli-1.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:2a24c50840d89ded6c9a8fdc7b6ed3692ed4e86f1c4a4a938e1e92def92933e0", size = 3034905 }, + { url = "https://files.pythonhosted.org/packages/84/9c/bc96b6c7db824998a49ed3b38e441a2cae9234da6fa11f6ed17e8cf4f147/Brotli-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f31859074d57b4639318523d6ffdca586ace54271a73ad23ad021acd807eb14b", size = 2929467 }, + { url = "https://files.pythonhosted.org/packages/e7/71/8f161dee223c7ff7fea9d44893fba953ce97cf2c3c33f78ba260a91bcff5/Brotli-1.1.0-cp311-cp311-win32.whl", hash = "sha256:39da8adedf6942d76dc3e46653e52df937a3c4d6d18fdc94a7c29d263b1f5b50", size = 333169 }, + { url = "https://files.pythonhosted.org/packages/02/8a/fece0ee1057643cb2a5bbf59682de13f1725f8482b2c057d4e799d7ade75/Brotli-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:aac0411d20e345dc0920bdec5548e438e999ff68d77564d5e9463a7ca9d3e7b1", size = 357253 }, + { url = "https://files.pythonhosted.org/packages/5c/d0/5373ae13b93fe00095a58efcbce837fd470ca39f703a235d2a999baadfbc/Brotli-1.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:32d95b80260d79926f5fab3c41701dbb818fde1c9da590e77e571eefd14abe28", size = 815693 }, + { url = "https://files.pythonhosted.org/packages/8e/48/f6e1cdf86751300c288c1459724bfa6917a80e30dbfc326f92cea5d3683a/Brotli-1.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b760c65308ff1e462f65d69c12e4ae085cff3b332d894637f6273a12a482d09f", size = 422489 }, + { url = "https://files.pythonhosted.org/packages/06/88/564958cedce636d0f1bed313381dfc4b4e3d3f6015a63dae6146e1b8c65c/Brotli-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409", size = 873081 }, + { url = "https://files.pythonhosted.org/packages/58/79/b7026a8bb65da9a6bb7d14329fd2bd48d2b7f86d7329d5cc8ddc6a90526f/Brotli-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2", size = 446244 }, + { url = "https://files.pythonhosted.org/packages/e5/18/c18c32ecea41b6c0004e15606e274006366fe19436b6adccc1ae7b2e50c2/Brotli-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451", size = 2906505 }, + { url = "https://files.pythonhosted.org/packages/08/c8/69ec0496b1ada7569b62d85893d928e865df29b90736558d6c98c2031208/Brotli-1.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7f4bf76817c14aa98cc6697ac02f3972cb8c3da93e9ef16b9c66573a68014f91", size = 2944152 }, + { url = "https://files.pythonhosted.org/packages/ab/fb/0517cea182219d6768113a38167ef6d4eb157a033178cc938033a552ed6d/Brotli-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0c5516f0aed654134a2fc936325cc2e642f8a0e096d075209672eb321cff408", size = 2919252 }, + { url = "https://files.pythonhosted.org/packages/c7/53/73a3431662e33ae61a5c80b1b9d2d18f58dfa910ae8dd696e57d39f1a2f5/Brotli-1.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c3020404e0b5eefd7c9485ccf8393cfb75ec38ce75586e046573c9dc29967a0", size = 2845955 }, + { url = "https://files.pythonhosted.org/packages/55/ac/bd280708d9c5ebdbf9de01459e625a3e3803cce0784f47d633562cf40e83/Brotli-1.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4ed11165dd45ce798d99a136808a794a748d5dc38511303239d4e2363c0695dc", size = 2914304 }, + { url = "https://files.pythonhosted.org/packages/76/58/5c391b41ecfc4527d2cc3350719b02e87cb424ef8ba2023fb662f9bf743c/Brotli-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180", size = 2814452 }, + { url = "https://files.pythonhosted.org/packages/c7/4e/91b8256dfe99c407f174924b65a01f5305e303f486cc7a2e8a5d43c8bec3/Brotli-1.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248", size = 2938751 }, + { url = "https://files.pythonhosted.org/packages/5a/a6/e2a39a5d3b412938362bbbeba5af904092bf3f95b867b4a3eb856104074e/Brotli-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966", size = 2933757 }, + { url = "https://files.pythonhosted.org/packages/13/f0/358354786280a509482e0e77c1a5459e439766597d280f28cb097642fc26/Brotli-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87a3044c3a35055527ac75e419dfa9f4f3667a1e887ee80360589eb8c90aabb9", size = 2936146 }, + { url = "https://files.pythonhosted.org/packages/80/f7/daf538c1060d3a88266b80ecc1d1c98b79553b3f117a485653f17070ea2a/Brotli-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c5529b34c1c9d937168297f2c1fde7ebe9ebdd5e121297ff9c043bdb2ae3d6fb", size = 2848055 }, + { url = "https://files.pythonhosted.org/packages/ad/cf/0eaa0585c4077d3c2d1edf322d8e97aabf317941d3a72d7b3ad8bce004b0/Brotli-1.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ca63e1890ede90b2e4454f9a65135a4d387a4585ff8282bb72964fab893f2111", size = 3035102 }, + { url = "https://files.pythonhosted.org/packages/d8/63/1c1585b2aa554fe6dbce30f0c18bdbc877fa9a1bf5ff17677d9cca0ac122/Brotli-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e79e6520141d792237c70bcd7a3b122d00f2613769ae0cb61c52e89fd3443839", size = 2930029 }, + { url = "https://files.pythonhosted.org/packages/5f/3b/4e3fd1893eb3bbfef8e5a80d4508bec17a57bb92d586c85c12d28666bb13/Brotli-1.1.0-cp312-cp312-win32.whl", hash = "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0", size = 333276 }, + { url = "https://files.pythonhosted.org/packages/3d/d5/942051b45a9e883b5b6e98c041698b1eb2012d25e5948c58d6bf85b1bb43/Brotli-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951", size = 357255 }, + { url = "https://files.pythonhosted.org/packages/0a/9f/fb37bb8ffc52a8da37b1c03c459a8cd55df7a57bdccd8831d500e994a0ca/Brotli-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8bf32b98b75c13ec7cf774164172683d6e7891088f6316e54425fde1efc276d5", size = 815681 }, + { url = "https://files.pythonhosted.org/packages/06/b3/dbd332a988586fefb0aa49c779f59f47cae76855c2d00f450364bb574cac/Brotli-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bc37c4d6b87fb1017ea28c9508b36bbcb0c3d18b4260fcdf08b200c74a6aee8", size = 422475 }, + { url = "https://files.pythonhosted.org/packages/bb/80/6aaddc2f63dbcf2d93c2d204e49c11a9ec93a8c7c63261e2b4bd35198283/Brotli-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c0ef38c7a7014ffac184db9e04debe495d317cc9c6fb10071f7fefd93100a4f", size = 2906173 }, + { url = "https://files.pythonhosted.org/packages/ea/1d/e6ca79c96ff5b641df6097d299347507d39a9604bde8915e76bf026d6c77/Brotli-1.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91d7cc2a76b5567591d12c01f019dd7afce6ba8cba6571187e21e2fc418ae648", size = 2943803 }, + { url = "https://files.pythonhosted.org/packages/ac/a3/d98d2472e0130b7dd3acdbb7f390d478123dbf62b7d32bda5c830a96116d/Brotli-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a93dde851926f4f2678e704fadeb39e16c35d8baebd5252c9fd94ce8ce68c4a0", size = 2918946 }, + { url = "https://files.pythonhosted.org/packages/c4/a5/c69e6d272aee3e1423ed005d8915a7eaa0384c7de503da987f2d224d0721/Brotli-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0db75f47be8b8abc8d9e31bc7aad0547ca26f24a54e6fd10231d623f183d089", size = 2845707 }, + { url = "https://files.pythonhosted.org/packages/58/9f/4149d38b52725afa39067350696c09526de0125ebfbaab5acc5af28b42ea/Brotli-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6967ced6730aed543b8673008b5a391c3b1076d834ca438bbd70635c73775368", size = 2936231 }, + { url = "https://files.pythonhosted.org/packages/5a/5a/145de884285611838a16bebfdb060c231c52b8f84dfbe52b852a15780386/Brotli-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7eedaa5d036d9336c95915035fb57422054014ebdeb6f3b42eac809928e40d0c", size = 2848157 }, + { url = "https://files.pythonhosted.org/packages/50/ae/408b6bfb8525dadebd3b3dd5b19d631da4f7d46420321db44cd99dcf2f2c/Brotli-1.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d487f5432bf35b60ed625d7e1b448e2dc855422e87469e3f450aa5552b0eb284", size = 3035122 }, + { url = "https://files.pythonhosted.org/packages/af/85/a94e5cfaa0ca449d8f91c3d6f78313ebf919a0dbd55a100c711c6e9655bc/Brotli-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832436e59afb93e1836081a20f324cb185836c617659b07b129141a8426973c7", size = 2930206 }, + { url = "https://files.pythonhosted.org/packages/c2/f0/a61d9262cd01351df22e57ad7c34f66794709acab13f34be2675f45bf89d/Brotli-1.1.0-cp313-cp313-win32.whl", hash = "sha256:43395e90523f9c23a3d5bdf004733246fba087f2948f87ab28015f12359ca6a0", size = 333804 }, + { url = "https://files.pythonhosted.org/packages/7e/c1/ec214e9c94000d1c1974ec67ced1c970c148aa6b8d8373066123fc3dbf06/Brotli-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:9011560a466d2eb3f5a6e4929cf4a09be405c64154e12df0dd72713f6500e32b", size = 358517 }, + { url = "https://files.pythonhosted.org/packages/1b/aa/aa6e0c9848ee4375514af0b27abf470904992939b7363ae78fc8aca8a9a8/Brotli-1.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5fb2ce4b8045c78ebbc7b8f3c15062e435d47e7393cc57c25115cfd49883747a", size = 873048 }, + { url = "https://files.pythonhosted.org/packages/ae/32/38bba1a8bef9ecb1cda08439fd28d7e9c51aff13b4783a4f1610da90b6c2/Brotli-1.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7905193081db9bfa73b1219140b3d315831cbff0d8941f22da695832f0dd188f", size = 446207 }, + { url = "https://files.pythonhosted.org/packages/3c/6a/14cc20ddc53efc274601c8195791a27cfb7acc5e5134e0f8c493a8b8821a/Brotli-1.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a77def80806c421b4b0af06f45d65a136e7ac0bdca3c09d9e2ea4e515367c7e9", size = 2903803 }, + { url = "https://files.pythonhosted.org/packages/9a/26/62b2d894d4e82d7a7f4e0bb9007a42bbc765697a5679b43186acd68d7a79/Brotli-1.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8dadd1314583ec0bf2d1379f7008ad627cd6336625d6679cf2f8e67081b83acf", size = 2941149 }, + { url = "https://files.pythonhosted.org/packages/a9/ca/00d55bbdd8631236c61777742d8a8454cf6a87eb4125cad675912c68bec7/Brotli-1.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:901032ff242d479a0efa956d853d16875d42157f98951c0230f69e69f9c09bac", size = 2672253 }, + { url = "https://files.pythonhosted.org/packages/e2/e6/4a730f6e5b5d538e92d09bc51bf69119914f29a222f9e1d65ae4abb27a4e/Brotli-1.1.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:22fc2a8549ffe699bfba2256ab2ed0421a7b8fadff114a3d201794e45a9ff578", size = 2757005 }, + { url = "https://files.pythonhosted.org/packages/cb/6b/8cf297987fe3c1bf1c87f0c0b714af2ce47092b8d307b9f6ecbc65f98968/Brotli-1.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ae15b066e5ad21366600ebec29a7ccbc86812ed267e4b28e860b8ca16a2bc474", size = 2910658 }, + { url = "https://files.pythonhosted.org/packages/2c/1f/be9443995821c933aad7159803f84ef4923c6f5b72c2affd001192b310fc/Brotli-1.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:949f3b7c29912693cee0afcf09acd6ebc04c57af949d9bf77d6101ebb61e388c", size = 2809728 }, + { url = "https://files.pythonhosted.org/packages/76/2f/213bab6efa902658c80a1247142d42b138a27ccdd6bade49ca9cd74e714a/Brotli-1.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:89f4988c7203739d48c6f806f1e87a1d96e0806d44f0fba61dba81392c9e474d", size = 2935043 }, + { url = "https://files.pythonhosted.org/packages/27/89/bbb14fa98e895d1e601491fba54a5feec167d262f0d3d537a3b0d4cd0029/Brotli-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:de6551e370ef19f8de1807d0a9aa2cdfdce2e85ce88b122fe9f6b2b076837e59", size = 2930639 }, + { url = "https://files.pythonhosted.org/packages/14/87/03a6d6e1866eddf9f58cc57e35befbeb5514da87a416befe820150cae63f/Brotli-1.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0737ddb3068957cf1b054899b0883830bb1fec522ec76b1098f9b6e0f02d9419", size = 2932834 }, + { url = "https://files.pythonhosted.org/packages/a4/d5/e5f85e04f75144d1a89421ba432def6bdffc8f28b04f5b7d540bbd03362c/Brotli-1.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4f3607b129417e111e30637af1b56f24f7a49e64763253bbc275c75fa887d4b2", size = 2845213 }, + { url = "https://files.pythonhosted.org/packages/99/bf/25ef07add7afbb1aacd4460726a1a40370dfd60c0810b6f242a6d3871d7e/Brotli-1.1.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:6c6e0c425f22c1c719c42670d561ad682f7bfeeef918edea971a79ac5252437f", size = 3031573 }, + { url = "https://files.pythonhosted.org/packages/55/22/948a97bda5c9dc9968d56b9ed722d9727778db43739cf12ef26ff69be94d/Brotli-1.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:494994f807ba0b92092a163a0a283961369a65f6cbe01e8891132b7a320e61eb", size = 2926885 }, + { url = "https://files.pythonhosted.org/packages/31/ba/e53d107399b535ef89deb6977dd8eae468e2dde7b1b74c6cbe2c0e31fda2/Brotli-1.1.0-cp39-cp39-win32.whl", hash = "sha256:f0d8a7a6b5983c2496e364b969f0e526647a06b075d034f3297dc66f3b360c64", size = 333171 }, + { url = "https://files.pythonhosted.org/packages/99/b3/f7b3af539f74b82e1c64d28685a5200c631cc14ae751d37d6ed819655627/Brotli-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:cdad5b9014d83ca68c25d2e9444e28e967ef16e80f6b436918c700c117a85467", size = 357258 }, +] + +[[package]] +name = "brotlicffi" +version = "1.1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/9d/70caa61192f570fcf0352766331b735afa931b4c6bc9a348a0925cc13288/brotlicffi-1.1.0.0.tar.gz", hash = "sha256:b77827a689905143f87915310b93b273ab17888fd43ef350d4832c4a71083c13", size = 465192 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/11/7b96009d3dcc2c931e828ce1e157f03824a69fb728d06bfd7b2fc6f93718/brotlicffi-1.1.0.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9b7ae6bd1a3f0df532b6d67ff674099a96d22bc0948955cb338488c31bfb8851", size = 453786 }, + { url = "https://files.pythonhosted.org/packages/d6/e6/a8f46f4a4ee7856fbd6ac0c6fb0dc65ed181ba46cd77875b8d9bbe494d9e/brotlicffi-1.1.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19ffc919fa4fc6ace69286e0a23b3789b4219058313cf9b45625016bf7ff996b", size = 2911165 }, + { url = "https://files.pythonhosted.org/packages/be/20/201559dff14e83ba345a5ec03335607e47467b6633c210607e693aefac40/brotlicffi-1.1.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9feb210d932ffe7798ee62e6145d3a757eb6233aa9a4e7db78dd3690d7755814", size = 2927895 }, + { url = "https://files.pythonhosted.org/packages/cd/15/695b1409264143be3c933f708a3f81d53c4a1e1ebbc06f46331decbf6563/brotlicffi-1.1.0.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84763dbdef5dd5c24b75597a77e1b30c66604725707565188ba54bab4f114820", size = 2851834 }, + { url = "https://files.pythonhosted.org/packages/b4/40/b961a702463b6005baf952794c2e9e0099bde657d0d7e007f923883b907f/brotlicffi-1.1.0.0-cp37-abi3-win32.whl", hash = "sha256:1b12b50e07c3911e1efa3a8971543e7648100713d4e0971b13631cce22c587eb", size = 341731 }, + { url = "https://files.pythonhosted.org/packages/1c/fa/5408a03c041114ceab628ce21766a4ea882aa6f6f0a800e04ee3a30ec6b9/brotlicffi-1.1.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:994a4f0681bb6c6c3b0925530a1926b7a189d878e6e5e38fae8efa47c5d9c613", size = 366783 }, + { url = "https://files.pythonhosted.org/packages/e5/3b/bd4f3d2bcf2306ae66b0346f5b42af1962480b200096ffc7abc3bd130eca/brotlicffi-1.1.0.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:2e4aeb0bd2540cb91b069dbdd54d458da8c4334ceaf2d25df2f4af576d6766ca", size = 397397 }, + { url = "https://files.pythonhosted.org/packages/54/10/1fd57864449360852c535c2381ee7120ba8f390aa3869df967c44ca7eba1/brotlicffi-1.1.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b7b0033b0d37bb33009fb2fef73310e432e76f688af76c156b3594389d81391", size = 379698 }, + { url = "https://files.pythonhosted.org/packages/e5/95/15aa422aa6450e6556e54a5fd1650ff59f470aed77ac739aa90ab63dc611/brotlicffi-1.1.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54a07bb2374a1eba8ebb52b6fafffa2afd3c4df85ddd38fcc0511f2bb387c2a8", size = 378635 }, + { url = "https://files.pythonhosted.org/packages/6c/a7/f254e13b2cb43337d6d99a4ec10394c134e41bfda8a2eff15b75627f4a3d/brotlicffi-1.1.0.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7901a7dc4b88f1c1475de59ae9be59799db1007b7d059817948d8e4f12e24e35", size = 385719 }, + { url = "https://files.pythonhosted.org/packages/72/a9/0971251c4427c14b2a827dba3d910d4d3330dabf23d4278bf6d06a978847/brotlicffi-1.1.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ce01c7316aebc7fce59da734286148b1d1b9455f89cf2c8a4dfce7d41db55c2d", size = 361760 }, + { url = "https://files.pythonhosted.org/packages/35/9b/e0b577351e1d9d5890e1a56900c4ceaaef783b807145cd229446a43cf437/brotlicffi-1.1.0.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1a807d760763e398bbf2c6394ae9da5815901aa93ee0a37bca5efe78d4ee3171", size = 397392 }, + { url = "https://files.pythonhosted.org/packages/4f/7f/a16534d28386f74781db8b4544a764cf955abae336379a76f50e745bb0ee/brotlicffi-1.1.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fa8ca0623b26c94fccc3a1fdd895be1743b838f3917300506d04aa3346fd2a14", size = 379695 }, + { url = "https://files.pythonhosted.org/packages/50/2a/699388b5e489726991132441b55aff0691dd73c49105ef220408a5ab98d6/brotlicffi-1.1.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3de0cf28a53a3238b252aca9fed1593e9d36c1d116748013339f0949bfc84112", size = 378629 }, + { url = "https://files.pythonhosted.org/packages/4a/3f/58254e7fbe6011bf043e4dcade0e16995a9f82b731734fad97220d201f42/brotlicffi-1.1.0.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6be5ec0e88a4925c91f3dea2bb0013b3a2accda6f77238f76a34a1ea532a1cb0", size = 385712 }, + { url = "https://files.pythonhosted.org/packages/40/16/2a29a625a6f74d13726387f83484dfaaf6fcdaafaadfbe26a0412ae268cc/brotlicffi-1.1.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d9eb71bb1085d996244439154387266fd23d6ad37161f6f52f1cd41dd95a3808", size = 361747 }, +] + [[package]] name = "cachetools" version = "5.5.0" @@ -546,6 +659,54 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442 }, ] +[[package]] +name = "inflate64" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/99/18f9940d4a3f2cabc4396a587ddf1bd93236bdb372d9e78e2b0365e40990/inflate64-1.0.0.tar.gz", hash = "sha256:3278827b803cf006a1df251f3e13374c7d26db779e5a33329cc11789b804bc2d", size = 895853 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/cf/06af80e81dd4bbb7e883291cf1726035d526f066a37c4ed4d4cd88a7a49d/inflate64-1.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a90c0bdf4a7ecddd8a64cc977181810036e35807f56b0bcacee9abb0fcfd18dc", size = 59418 }, + { url = "https://files.pythonhosted.org/packages/c9/4b/6f18918220b1a8e935121cece1dc917e62fa593fc637a621470f9b9a601a/inflate64-1.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:57fe7c14aebf1c5a74fc3b70d355be1280a011521a76aa3895486e62454f4242", size = 36231 }, + { url = "https://files.pythonhosted.org/packages/aa/f4/f4b5dbd78dd5af66b6ca32778ebaa9c14d67b68ea84e96592ccf40786a41/inflate64-1.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d90730165f471d61a1a694a5e354f3ffa938227e8dcecb62d5d728e8069cee94", size = 35738 }, + { url = "https://files.pythonhosted.org/packages/10/23/26289a700550767cf5eb7550f78ad826529706287393f224bbaee3c1b1e2/inflate64-1.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:543f400201f5c101141af3c79c82059e1aa6ef4f1584a7f1fa035fb2e465097f", size = 92855 }, + { url = "https://files.pythonhosted.org/packages/b8/f4/e387a50f5027194eac4f9712d57b97e3e1a012402eaae98bcf1ebe8a97d1/inflate64-1.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ceca14f7ec19fb44b047f56c50efb7521b389d222bba2b0a10286a0caeb03fa", size = 93141 }, + { url = "https://files.pythonhosted.org/packages/33/c8/e516aecd9ed0dc75d8df041ed4ef80f2e2be39d0e516c7269b7f274e760a/inflate64-1.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b559937a42f0c175b4d2dfc7eb53b97bdc87efa9add15ed5549c6abc1e89d02f", size = 95262 }, + { url = "https://files.pythonhosted.org/packages/0b/aa/ed3ab5f8c13afc432fb382edf97cede7a6f9be73ecf98bfe64b686c8d223/inflate64-1.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5ff8bd2a562343fcbc4eea26fdc368904a3b5f6bb8262344274d3d74a1de15bb", size = 95912 }, + { url = "https://files.pythonhosted.org/packages/e0/64/5637c4f67ed15518c0765b85b528ed79536caaf8ba167a9f7173e334d4a8/inflate64-1.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:0fe481f31695d35a433c3044ac8fd5d9f5069aaad03a0c04b570eb258ce655aa", size = 35166 }, + { url = "https://files.pythonhosted.org/packages/af/92/701b3c76b1cf244026c3e78dff8487955cf6960c1d9f350e2820a0d1a5d9/inflate64-1.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a45f6979ad5874d4d4898c2fc770b136e61b96b850118fdaec5a5af1b9123a", size = 59450 }, + { url = "https://files.pythonhosted.org/packages/bb/1d/af0253fafc27cadd29e3b111ebb3011b8c913a3554b403c90c7595f5933e/inflate64-1.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:022ca1cc928e7365a05f7371ff06af143c6c667144965e2cf9a9236a2ae1c291", size = 36267 }, + { url = "https://files.pythonhosted.org/packages/b6/22/7949030be11f4754bd6ed7067e9bebdf614013b89ccd4638330a85821b51/inflate64-1.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:46792ecf3565d64fd2c519b0a780c03a57e195613c9954ef94e739a057b3fd06", size = 35740 }, + { url = "https://files.pythonhosted.org/packages/e4/87/c6ce0093a345c04811f6171a367665dec17dcc4617ca150dd37e9ae7bd33/inflate64-1.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a70ea2e456c15f7aa7c74b8ab8f20b4f8940ec657604c9f0a9de3342f280fff", size = 95896 }, + { url = "https://files.pythonhosted.org/packages/62/d6/fe113b12773cad2c093d381c2b1629f9cfa240c9ad86a7f9f9079e7a51b5/inflate64-1.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e243ea9bd36a035059f2365bd6d156ff59717fbafb0255cb0c75bf151bf6904", size = 96007 }, + { url = "https://files.pythonhosted.org/packages/f0/a6/9165bee4b7fc5af949fec12a2cea7ad73bf9ee97dfb96a0276274c48e709/inflate64-1.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4dc392dec1cd11cacda3d2637214ca45e38202e8a4f31d4a4e566d6e90625fc4", size = 98297 }, + { url = "https://files.pythonhosted.org/packages/ee/72/0aeb360101eeed32696fc6c623bc1780fac895a9fc2e93b582cb1e22ca54/inflate64-1.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8b402a50eda7ee75f342fc346d33a41bca58edc222a4b17f9be0db1daed459fa", size = 98858 }, + { url = "https://files.pythonhosted.org/packages/94/4a/8301ad59b57d9de504b0fdce22bf980dfb231753e6d7aed12af938f7f9fd/inflate64-1.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:f5924499dc8800928c0ee4580fa8eb4ffa880b2cce4431537d0390e503a9c9ee", size = 35167 }, + { url = "https://files.pythonhosted.org/packages/18/82/47021b8919c1dc276d0502296f15ffac1cd648b94b35cadb14cb812b6199/inflate64-1.0.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0c644bf7208e20825ca3bbb5fb1f7f495cfcb49eb01a5f67338796d44a42f2bf", size = 59509 }, + { url = "https://files.pythonhosted.org/packages/e0/c9/00701be8e48dc9c9b9488001d9c66d6cb6f6bb0c48af9abf33a69726d130/inflate64-1.0.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9964a4eaf26a9d36f82a1d9b12c28e35800dd3d99eb340453ed12ac90c2976a8", size = 36305 }, + { url = "https://files.pythonhosted.org/packages/25/c0/11dea5e298b2e7d61f0fbd1005553e8796e35536751980b676547fcc57ef/inflate64-1.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2cccded63865640d03253897be7232b2bbac295fe43914c61f86a57aa23bb61d", size = 35756 }, + { url = "https://files.pythonhosted.org/packages/86/ba/4debdaaafdc21853621caf463a498a754ee4352893454c596dbd65294e9f/inflate64-1.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d491f104fb3701926ebd82b8c9250dfba0ddcab584504e26f1e4adb26730378d", size = 96127 }, + { url = "https://files.pythonhosted.org/packages/89/81/8f559c199ec13d0b70d0dc46811490b2976873c96c564941583777e9b343/inflate64-1.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ebad4a6cd2a2c1d81be0b09d4006479f3b258803c49a9224ef8ca0b649072fa", size = 96903 }, + { url = "https://files.pythonhosted.org/packages/46/41/39ac4c7e17d0690578b716a0ff34e00600616994795b0645fd61fc600c0f/inflate64-1.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:6823b2c0cff3a8159140f3b17ec64fb8ec0e663b45a6593618ecdde8aeecb5b2", size = 98855 }, + { url = "https://files.pythonhosted.org/packages/44/dd/be5d69492c180f94a6af8a15564ce365bdcb84bd1a6fb32949d6913959aa/inflate64-1.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:228d504239d27958e71fc77e3119a6ac4528127df38468a0c95a5bd3927204b8", size = 99884 }, + { url = "https://files.pythonhosted.org/packages/8c/0d/a5266bd4f2cdb7fad1eae3ffe4dcc16f9769323660a0a6cfbe9cc1d2cf03/inflate64-1.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:ae2572e06bcfe15e3bbf77d4e4a6d6c55e2a70d6abceaaf60c5c3653ddb96dfd", size = 35334 }, + { url = "https://files.pythonhosted.org/packages/04/1c/47f9c93df339c381f3f3e7b983d7abf0756f8bd227bf9fb5a2e8b09ea9e5/inflate64-1.0.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bf2981b95c1f26242bb084d9a07f3feb0cfe3d6d0a8d90f42389803bc1252c4a", size = 59410 }, + { url = "https://files.pythonhosted.org/packages/b9/28/bde1595cae0379c521ae74f698f7da1345d990d2df7558df59470ced639b/inflate64-1.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9373ccf0661cc72ac84a0ad622634144da5ce7d57c9572ed0723d67a149feed2", size = 36230 }, + { url = "https://files.pythonhosted.org/packages/13/64/4350547de7c8a3ac27c1a0ab5807ac2fcbcde121c44190f87da1039421eb/inflate64-1.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e4650c6f65011ec57cf5cd96b92d5b7c6f59e502930c86eb8227c93cf02dc270", size = 35739 }, + { url = "https://files.pythonhosted.org/packages/1a/d6/b55e43c3503d7780aabe3d1d5493c983a99947dff228d9b622d49c58af68/inflate64-1.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a475e8822f1a74c873e60b8f270773757ade024097ca39e43402d47c049c67d4", size = 92655 }, + { url = "https://files.pythonhosted.org/packages/c0/c0/417e5183543445818930b3fe181d718e519d26a227b5b77871d8f0c8502d/inflate64-1.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4367480733ac8daf368f6fc704b7c9db85521ee745eb5bd443f4b97d2051acc", size = 92966 }, + { url = "https://files.pythonhosted.org/packages/6d/8e/ada659c83abb78222c666bb8d35b4791cca25a8a4a750f4bc457402d2430/inflate64-1.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6c5775c91f94f5eced9160fb0af12a09f3e030194f91a6a46e706a79350bd056", size = 95075 }, + { url = "https://files.pythonhosted.org/packages/7f/15/59c0e45e091599a05a098374166ff74d3da5f9c0ffa72e8c78b7f0fcaa2b/inflate64-1.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d76d205b844d78ce04768060084ef20e64dcc63a3e9166674f857acaf4d140ed", size = 95751 }, + { url = "https://files.pythonhosted.org/packages/80/21/a90d085b9ea67729d5cef57566c69fd4dec8205c22415cde4be62b7adc64/inflate64-1.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:92f0dc6af0e8e97324981178dc442956cbff1247a56d1e201af8d865244653f8", size = 35164 }, + { url = "https://files.pythonhosted.org/packages/53/91/43238dd8a7e5bab71abae872c09931db4b31aebf672afccb305f79aacb3e/inflate64-1.0.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f79542478e49e471e8b23556700e6f688a40dc93e9a746f77a546c13251b59b1", size = 34648 }, + { url = "https://files.pythonhosted.org/packages/ef/6f/ce090934a80c1fd0b5b07c125ed6eb2845f11a78af344d69c0f051dcab97/inflate64-1.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a270be6b10cde01258c0097a663a307c62d12c78eb8f62f8e29f205335942c9", size = 36473 }, + { url = "https://files.pythonhosted.org/packages/b4/fe/2cd4bf78696213b807860002c182dd1751ba52c1559143b1b8daa7904733/inflate64-1.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1616a87ff04f583e9558cc247ec0b72a30d540ee0c17cc77823be175c0ec92f0", size = 36478 }, + { url = "https://files.pythonhosted.org/packages/43/dd/e62444c0ef7d1228b622e6d3dacf9ea237d8807a78619a83832a3b4a5adf/inflate64-1.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:137ca6b315f0157a786c3a755a09395ca69aed8bcf42ad3437cb349f5ebc86d2", size = 35630 }, + { url = "https://files.pythonhosted.org/packages/2b/1e/0e346cb5eced6f2f30bb110e6d7c7ee7ab70ad1ff44b743d4cf314f60f7d/inflate64-1.0.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:dd6d3e7d47df43210a995fd1f5989602b64de3f2a17cf4cbff553518b3577fd4", size = 34644 }, + { url = "https://files.pythonhosted.org/packages/d4/f7/7ac502391ea56af23661a707e9c94efc01376a27c1b06f7a0cfb553c0f17/inflate64-1.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f033b2879696b855200cde5ca4e293132c7499df790acb2c0dacb336d5e83b1", size = 36467 }, + { url = "https://files.pythonhosted.org/packages/09/ea/7c3e7c856f42bee178a9fea816e259b02bd4198873cf07b5a839b96d9696/inflate64-1.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f816d1c8a0593375c289e285c96deaee9c2d8742cb0edbd26ee05588a9ae657", size = 36475 }, + { url = "https://files.pythonhosted.org/packages/ae/b1/954207a9bd259a57c8c71ba0a26739a32f0d6227529abc9412751f3a87d2/inflate64-1.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1facd35319b6a391ee4c3d709c7c650bcada8cd7141d86cd8c2257287f45e6e6", size = 35626 }, +] + [[package]] name = "iniconfig" version = "2.0.0" @@ -672,6 +833,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f0/74/c95adcdf032956d9ef6c89a9b8a5152bf73915f8c633f3e3d88d06bd699c/mistune-3.0.2-py3-none-any.whl", hash = "sha256:71481854c30fdbc938963d3605b72501f5c10a9320ecd412c121c163a1c7d205", size = 47958 }, ] +[[package]] +name = "multivolumefile" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/50/f0/a7786212b5a4cb9ba05ae84a2bbd11d1d0279523aea0424b6d981d652a14/multivolumefile-0.2.3.tar.gz", hash = "sha256:a0648d0aafbc96e59198d5c17e9acad7eb531abea51035d08ce8060dcad709d6", size = 77984 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/31/ec5f46fd4c83185b806aa9c736e228cb780f13990a9cf4da0beb70025fcc/multivolumefile-0.2.3-py3-none-any.whl", hash = "sha256:237f4353b60af1703087cf7725755a1f6fcaeeea48421e1896940cd1c920d678", size = 17037 }, +] + [[package]] name = "mypy-extensions" version = "1.0.0" @@ -808,7 +978,7 @@ dev = [ { name = "pyright", specifier = "==1.1.373" }, { name = "pytest", specifier = ">=6.2.5" }, { name = "pytest-cov", specifier = ">=2.12.1" }, - { name = "ruff", specifier = "==0.1.6" }, + { name = "ruff", specifier = "==0.8.2" }, { name = "watchdog", specifier = ">=2.1.5" }, ] @@ -823,6 +993,7 @@ dependencies = [ { name = "gdown" }, { name = "gsheets" }, { name = "pandas" }, + { name = "py7zr" }, { name = "pyarrow" }, { name = "pydrive2" }, { name = "structlog" }, @@ -837,6 +1008,7 @@ requires-dist = [ { name = "gdown", specifier = ">=4.5.2" }, { name = "gsheets", specifier = ">=0.6.1" }, { name = "pandas", specifier = ">=2.2.1" }, + { name = "py7zr", specifier = ">=0.22.0" }, { name = "pyarrow", specifier = ">=10.0.1" }, { name = "pydrive2", specifier = ">=1.15.0" }, { name = "structlog", specifier = ">=21.5.0" }, @@ -855,7 +1027,7 @@ dev = [ { name = "pyright", specifier = "==1.1.373" }, { name = "pytest", specifier = ">=6.2.5" }, { name = "pytest-cov", specifier = ">=2.12.1" }, - { name = "ruff", specifier = "==0.1.6" }, + { name = "ruff", specifier = "==0.8.2" }, { name = "sphinx", specifier = "==4.5.0" }, { name = "watchdog", specifier = ">=2.1.5" }, ] @@ -882,7 +1054,7 @@ dev = [ { name = "ipdb", specifier = ">=0.13.13" }, { name = "pyright", specifier = "==1.1.373" }, { name = "pytest", specifier = ">=7.2.0" }, - { name = "ruff", specifier = "==0.1.6" }, + { name = "ruff", specifier = "==0.8.2" }, ] [[package]] @@ -1028,6 +1200,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/3d/71fae0078424ba8ea70b222b6fa56ef771a9918ab91cee806c2abc9d57fa/protobuf-5.28.1-py3-none-any.whl", hash = "sha256:c529535e5c0effcf417682563719e5d8ac8d2b93de07a56108b4c2d436d7a29a", size = 169572 }, ] +[[package]] +name = "psutil" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/10/2a30b13c61e7cf937f4adf90710776b7918ed0a9c434e2c38224732af310/psutil-6.1.0.tar.gz", hash = "sha256:353815f59a7f64cdaca1c0307ee13558a0512f6db064e92fe833784f08539c7a", size = 508565 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/2b/f4dea5d993d9cd22ad958eea828a41d5d225556123d372f02547c29c4f97/psutil-6.1.0-cp27-none-win32.whl", hash = "sha256:9118f27452b70bb1d9ab3198c1f626c2499384935aaf55388211ad982611407e", size = 246648 }, + { url = "https://files.pythonhosted.org/packages/9f/14/4aa97a7f2e0ac33a050d990ab31686d651ae4ef8c86661fef067f00437b9/psutil-6.1.0-cp27-none-win_amd64.whl", hash = "sha256:a8506f6119cff7015678e2bce904a4da21025cc70ad283a53b099e7620061d85", size = 249905 }, + { url = "https://files.pythonhosted.org/packages/01/9e/8be43078a171381953cfee33c07c0d628594b5dbfc5157847b85022c2c1b/psutil-6.1.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6e2dcd475ce8b80522e51d923d10c7871e45f20918e027ab682f94f1c6351688", size = 247762 }, + { url = "https://files.pythonhosted.org/packages/1d/cb/313e80644ea407f04f6602a9e23096540d9dc1878755f3952ea8d3d104be/psutil-6.1.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:0895b8414afafc526712c498bd9de2b063deaac4021a3b3c34566283464aff8e", size = 248777 }, + { url = "https://files.pythonhosted.org/packages/65/8e/bcbe2025c587b5d703369b6a75b65d41d1367553da6e3f788aff91eaf5bd/psutil-6.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9dcbfce5d89f1d1f2546a2090f4fcf87c7f669d1d90aacb7d7582addece9fb38", size = 284259 }, + { url = "https://files.pythonhosted.org/packages/58/4d/8245e6f76a93c98aab285a43ea71ff1b171bcd90c9d238bf81f7021fb233/psutil-6.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:498c6979f9c6637ebc3a73b3f87f9eb1ec24e1ce53a7c5173b8508981614a90b", size = 287255 }, + { url = "https://files.pythonhosted.org/packages/27/c2/d034856ac47e3b3cdfa9720d0e113902e615f4190d5d1bdb8df4b2015fb2/psutil-6.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d905186d647b16755a800e7263d43df08b790d709d575105d419f8b6ef65423a", size = 288804 }, + { url = "https://files.pythonhosted.org/packages/ea/55/5389ed243c878725feffc0d6a3bc5ef6764312b6fc7c081faaa2cfa7ef37/psutil-6.1.0-cp37-abi3-win32.whl", hash = "sha256:1ad45a1f5d0b608253b11508f80940985d1d0c8f6111b5cb637533a0e6ddc13e", size = 250386 }, + { url = "https://files.pythonhosted.org/packages/11/91/87fa6f060e649b1e1a7b19a4f5869709fbf750b7c8c262ee776ec32f3028/psutil-6.1.0-cp37-abi3-win_amd64.whl", hash = "sha256:a8fb3752b491d246034fa4d279ff076501588ce8cbcdbb62c32fd7a377d996be", size = 254228 }, +] + [[package]] name = "ptyprocess" version = "0.7.0" @@ -1046,6 +1235,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842 }, ] +[[package]] +name = "py7zr" +version = "0.22.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "brotli", marker = "platform_python_implementation == 'CPython'" }, + { name = "brotlicffi", marker = "platform_python_implementation == 'PyPy'" }, + { name = "inflate64" }, + { name = "multivolumefile" }, + { name = "psutil", marker = "sys_platform != 'cygwin'" }, + { name = "pybcj" }, + { name = "pycryptodomex" }, + { name = "pyppmd" }, + { name = "pyzstd" }, + { name = "texttable" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/c3/0e05c711c16af0b9c47f3f77323303b338b9a871ba020d95d2b8dd6605ae/py7zr-0.22.0.tar.gz", hash = "sha256:c6c7aea5913535184003b73938490f9a4d8418598e533f9ca991d3b8e45a139e", size = 4992926 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/59/dd1750002c0f46099281116f8165247bc62dc85edad41cdd26e7b26de19d/py7zr-0.22.0-py3-none-any.whl", hash = "sha256:993b951b313500697d71113da2681386589b7b74f12e48ba13cc12beca79d078", size = 67906 }, +] + [[package]] name = "pyarrow" version = "17.0.0" @@ -1106,6 +1316,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/89/bc88a6711935ba795a679ea6ebee07e128050d6382eaa35a0a47c8032bdc/pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd", size = 181537 }, ] +[[package]] +name = "pybcj" +version = "1.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/d2/22e808b9d25ce3b43f5c8a9e22d873d403485ba55d84a4d6d5d044881762/pybcj-1.0.2.tar.gz", hash = "sha256:c7f5bef7f47723c53420e377bc64d2553843bee8bcac5f0ad076ab1524780018", size = 2111002 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/93/4735636b5905b7597068a2c7a10a8df0f668f28659207c274d64a4468b97/pybcj-1.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7bff28d97e47047d69a4ac6bf59adda738cf1d00adde8819117fdb65d966bdbc", size = 32556 }, + { url = "https://files.pythonhosted.org/packages/a6/37/443cd704397b6df54ff0822032e4815aca4e9badabc5ce1faac34235a40c/pybcj-1.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:198e0b4768b4025eb3309273d7e81dc53834b9a50092be6e0d9b3983cfd35c35", size = 23751 }, + { url = "https://files.pythonhosted.org/packages/9a/aa/5a19ed8661e979a4d3237a11706f9a16a474a2227fdd99ccb284be100a98/pybcj-1.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fa26415b4a118ea790de9d38f244312f2510a9bb5c65e560184d241a6f391a2d", size = 23980 }, + { url = "https://files.pythonhosted.org/packages/fe/5f/638ce03948905d267c8c0ccab81b8b4943a0324f63d8bdb0a0e2a85d4503/pybcj-1.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fabb2be57e4ca28ea36c13146cdf97d73abd27c51741923fc6ba1e8cd33e255c", size = 50155 }, + { url = "https://files.pythonhosted.org/packages/09/70/8b6a6cc2a5721f67f629bdc17875c0d603d57f360a19b099a7b4de19383d/pybcj-1.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75d6d613bae6f27678d5e44e89d61018779726aa6aa950c516d33a04b8af8c59", size = 49729 }, + { url = "https://files.pythonhosted.org/packages/89/06/2e41e34da0bb2adb3644cbf4366c344e5804a10f1153da7b3a23333f7db8/pybcj-1.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3ffae79ef8a1ea81ea2748ad7b7ad9b882aa88ddf65ce90f9e944df639eccc61", size = 54310 }, + { url = "https://files.pythonhosted.org/packages/b5/0f/de9e76c305d4dcd9d428a90ccac030f06c780bc30549fc449a944a6321bc/pybcj-1.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bdb4d8ff5cba3e0bd1adee7d20dbb2b4d80cb31ac04d6ea1cd06cfc02d2ecd0d", size = 53679 }, + { url = "https://files.pythonhosted.org/packages/1a/41/a807ff6b77ec8e49c749ed1d0db5649fbb1150c6fb5fb391115f4f1d743a/pybcj-1.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a29be917fbc99eca204b08407e0971e0205bfdad4b74ec915930675f352b669d", size = 24690 }, + { url = "https://files.pythonhosted.org/packages/27/0a/20bf70a7eb7c6b2668ff2af798254033c32a09d6c58ec9a87cd6aa843df5/pybcj-1.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a2562ebe5a0abec4da0229f8abb5e90ee97b178f19762eb925c1159be36828b3", size = 32581 }, + { url = "https://files.pythonhosted.org/packages/a9/b6/43977fe4296d2778c6dc67b596bb6a851eaea80f3dd4ff454e5fca8142c2/pybcj-1.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:af19bc61ded933001cd68f004ae2042bf1a78eb498a3c685ebd655fa1be90dbe", size = 23767 }, + { url = "https://files.pythonhosted.org/packages/89/c7/a61010f59406b8a45bb4865faa4b61d6b177dcfac04247fb56c7538d997d/pybcj-1.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f3f4a447800850aba7724a2274ea0a4800724520c1caf38f7d0dabf2f89a5e15", size = 23976 }, + { url = "https://files.pythonhosted.org/packages/10/7a/78848edbb6f12d9b86e375fc46135d9a204ededbf96682b05cb4b4fbd942/pybcj-1.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce1c8af7a4761d2b1b531864d84113948daa0c4245775c63bd9874cb955f4662", size = 51246 }, + { url = "https://files.pythonhosted.org/packages/9e/13/af86c86cdfb293e82dd0b6c4bbdf08645cd8993456ee3fb911c3eeed1b22/pybcj-1.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8007371f6f2b462f5aa05d5c2135d0a1bcf5b7bdd9bd15d86c730f588d10b7d3", size = 50754 }, + { url = "https://files.pythonhosted.org/packages/39/52/88600aa374b100612a1d82fca4b03eb4315e0084a05ee314ba1b771f7190/pybcj-1.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1079ca63ff8da5c936b76863690e0bd2489e8d4e0a3a340e032095dae805dd91", size = 55334 }, + { url = "https://files.pythonhosted.org/packages/56/67/3cf9747ef5b53e16a844217c6c9840be6289d05ec785500da2cc55cc25f2/pybcj-1.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e9a785eb26884429d9b9f6326e68c3638828c83bf6d42d2463c97ad5385caff2", size = 54714 }, + { url = "https://files.pythonhosted.org/packages/78/81/a71197903b503f54b85f4d352f909e701e9d26953577bd34d3fbe0520d5d/pybcj-1.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:9ea46e2d45469d13b7f25b08efcdb140220bab1ac5a850db0954591715b8caaa", size = 24693 }, + { url = "https://files.pythonhosted.org/packages/83/60/a3b43836895654aa93b5a8422adc3717359db98da9147abfabffef79f1e7/pybcj-1.0.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:21b5f2460629167340403d359289a173e0729ce8e84e3ce99462009d5d5e01a4", size = 32677 }, + { url = "https://files.pythonhosted.org/packages/50/b9/96c8d9577b0f5a701e4497408e6a331a08eb902aca8dfd4c5bb1eaab4779/pybcj-1.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:2940fb85730b9869254559c491cd83cf777e56c76a8a60df60e4be4f2a4248d7", size = 23813 }, + { url = "https://files.pythonhosted.org/packages/b7/1a/c80132feb084ec4098c0315a132799bddda8878113b5f956e21c4377f5f1/pybcj-1.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f40f3243139d675f43793a4e35c410c370f7b91ccae74e70c8b2f4877869f90e", size = 24019 }, + { url = "https://files.pythonhosted.org/packages/b1/94/62c3bf8a60b4787b46e21f43277d9cb8b6037c8ee183450f035a19a2bc4b/pybcj-1.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c2b3e60b65c7ac73e44335934e1e122da8d56db87840984601b3c5dc0ae4c19", size = 51927 }, + { url = "https://files.pythonhosted.org/packages/8b/9e/4ebd092251ef8d15408388be508617d5949cbba4baa2a6cfbb7e0a9b62c0/pybcj-1.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:746550dc7b5af4d04bb5fa4d065f18d39c925bcb5dee30db75747cd9a58bb6e8", size = 51665 }, + { url = "https://files.pythonhosted.org/packages/24/ea/da4637563468854bd361a69cd883946015f54fa119a5d9c655d26f151954/pybcj-1.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:8ce9b62b6aaa5b08773be8a919ecc4e865396c969f982b685eeca6e80c82abb7", size = 56041 }, + { url = "https://files.pythonhosted.org/packages/cf/b2/9b9e670818af925ed9a0168a5c021ccfcc089637d0e6651d16fd05896425/pybcj-1.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:493eab2b1f6f546730a6de0c5ceb75ce16f3767154e8ae30e2b70d41b928b7d2", size = 55606 }, + { url = "https://files.pythonhosted.org/packages/72/e9/d6b1bdf3a5aca8f3981145a5228ad51d72e2477a55927604a4768765e915/pybcj-1.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:ef55b96b7f2ed823e0b924de902065ec42ade856366c287dbb073fabd6b90ec1", size = 24719 }, + { url = "https://files.pythonhosted.org/packages/6e/18/ca43a186a570b3f6820a24f3ad726d0f0322f0b08f5550a92f99741a2e58/pybcj-1.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:fdb7cd8271471a5979d84915c1ee57eea7e0a69c893225fc418db66883b0e2a7", size = 32548 }, + { url = "https://files.pythonhosted.org/packages/f1/10/ee383b3450f7f13fbe234668fcf143b9a1e916e0cb84f5267ff3a5c6ad60/pybcj-1.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e96ae14062bdcddc3197300e6ee4efa6fbc6749be917db934eac66d0daaecb68", size = 23743 }, + { url = "https://files.pythonhosted.org/packages/1f/90/2d2851d694a7dc3640c6309af0e534cab145e7463bc408114db7eaa20115/pybcj-1.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a54ebdc8423ba99d75372708a882fcfc3b14d9d52cf195295ad53e5a47dab37f", size = 23971 }, + { url = "https://files.pythonhosted.org/packages/9a/29/8326dbfea26d643d5a95f836103ac278eb297143d881188d94b987e3a520/pybcj-1.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3602be737c6e9553c45ae89e6b0e556f64f34dabf27d5260317d1824d31b79d3", size = 49937 }, + { url = "https://files.pythonhosted.org/packages/ac/b2/26fa2cba6bc488380515929757cafbdbf01f30184a1aa11ef7ee35bb21a2/pybcj-1.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63dd2ca52a48841f561bfec0fa3f208d375b0a8dcd3d7b236459e683ae29221d", size = 49517 }, + { url = "https://files.pythonhosted.org/packages/58/7b/d5e39a73202eb7b67793a4313ae5a85bdbf1470899dc2d3119c6a2414e9b/pybcj-1.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8204a714029784b1a08a3d790430d80b423b68615c5b1e67aabca5bd5419b77d", size = 54141 }, + { url = "https://files.pythonhosted.org/packages/8e/f0/5abc858fe9d07338e485e86b2d82d1f1a0aa36c2af5271156c1140d04d15/pybcj-1.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:fde2376b180ae2620c102fbc3ef06638d306feae83964aaa5051ecbdda54845a", size = 53499 }, + { url = "https://files.pythonhosted.org/packages/9f/ac/4bad26429aab693235035f813dc60ff00ff5164acbbb98c4e26f190a21cd/pybcj-1.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:3b8d7810fb587adbffba025330cf212d9bbed8f29559656d05cb6609673f306a", size = 24686 }, +] + [[package]] name = "pycparser" version = "2.22" @@ -1115,6 +1365,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552 }, ] +[[package]] +name = "pycryptodomex" +version = "3.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/dc/e66551683ade663b5f07d7b3bc46434bf703491dbd22ee12d1f979ca828f/pycryptodomex-3.21.0.tar.gz", hash = "sha256:222d0bd05381dd25c32dd6065c071ebf084212ab79bab4599ba9e6a3e0009e6c", size = 4818543 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/5e/99f217d9881eead69607a2248dd7bbdf610837d7f5ad53f45a6cb71bbbfb/pycryptodomex-3.21.0-cp36-abi3-macosx_10_9_universal2.whl", hash = "sha256:34325b84c8b380675fd2320d0649cdcbc9cf1e0d1526edbe8fce43ed858cdc7e", size = 2499490 }, + { url = "https://files.pythonhosted.org/packages/ce/8f/4d0e2a859a6470289d64e39b419f01d2494dfa2e4995342d50f6c2834237/pycryptodomex-3.21.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:103c133d6cd832ae7266feb0a65b69e3a5e4dbbd6f3a3ae3211a557fd653f516", size = 1638037 }, + { url = "https://files.pythonhosted.org/packages/0c/9e/6e748c1fa814c956d356f93cf7192b19487ca56fc9e2a0bcde2bbc057601/pycryptodomex-3.21.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77ac2ea80bcb4b4e1c6a596734c775a1615d23e31794967416afc14852a639d3", size = 2172279 }, + { url = "https://files.pythonhosted.org/packages/46/3f/f5bef92b11750af9e3516d4e69736eeeff20a2818d34611508bef5a7b381/pycryptodomex-3.21.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9aa0cf13a1a1128b3e964dc667e5fe5c6235f7d7cfb0277213f0e2a783837cc2", size = 2258130 }, + { url = "https://files.pythonhosted.org/packages/de/4d/f0c65afd64ce435fd0547187ce6f99dfb37cdde16b05b57bca9f5c06966e/pycryptodomex-3.21.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:46eb1f0c8d309da63a2064c28de54e5e614ad17b7e2f88df0faef58ce192fc7b", size = 2297719 }, + { url = "https://files.pythonhosted.org/packages/1c/6a/2a1a101b0345ee70376ba93df8de6c8c01aac8341fda02970800873456a7/pycryptodomex-3.21.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:cc7e111e66c274b0df5f4efa679eb31e23c7545d702333dfd2df10ab02c2a2ce", size = 2164079 }, + { url = "https://files.pythonhosted.org/packages/3d/00/90a15f16c234815b660303c2d7266b41b401ea2605f3a90373e9d425e39f/pycryptodomex-3.21.0-cp36-abi3-musllinux_1_2_i686.whl", hash = "sha256:770d630a5c46605ec83393feaa73a9635a60e55b112e1fb0c3cea84c2897aa0a", size = 2333060 }, + { url = "https://files.pythonhosted.org/packages/61/74/49f5d20c514ccc631b940cc9dfec45dcce418dc84a98463a2e2ebec33904/pycryptodomex-3.21.0-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:52e23a0a6e61691134aa8c8beba89de420602541afaae70f66e16060fdcd677e", size = 2257982 }, + { url = "https://files.pythonhosted.org/packages/92/4b/d33ef74e2cc0025a259936661bb53432c5bbbadc561c5f2e023bcd73ce4c/pycryptodomex-3.21.0-cp36-abi3-win32.whl", hash = "sha256:a3d77919e6ff56d89aada1bd009b727b874d464cb0e2e3f00a49f7d2e709d76e", size = 1779052 }, + { url = "https://files.pythonhosted.org/packages/5b/be/7c991840af1184009fc86267160948350d1bf875f153c97bb471ad944e40/pycryptodomex-3.21.0-cp36-abi3-win_amd64.whl", hash = "sha256:b0e9765f93fe4890f39875e6c90c96cb341767833cfa767f41b490b506fa9ec0", size = 1816307 }, + { url = "https://files.pythonhosted.org/packages/af/ac/24125ad36778914a36f08d61ba5338cb9159382c638d9761ee19c8de822c/pycryptodomex-3.21.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:feaecdce4e5c0045e7a287de0c4351284391fe170729aa9182f6bd967631b3a8", size = 1694999 }, + { url = "https://files.pythonhosted.org/packages/93/73/be7a54a5903508070e5508925ba94493a1f326cfeecfff750e3eb250ea28/pycryptodomex-3.21.0-pp27-pypy_73-win32.whl", hash = "sha256:365aa5a66d52fd1f9e0530ea97f392c48c409c2f01ff8b9a39c73ed6f527d36c", size = 1769437 }, + { url = "https://files.pythonhosted.org/packages/e5/9f/39a6187f3986841fa6a9f35c6fdca5030ef73ff708b45a993813a51d7d10/pycryptodomex-3.21.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3efddfc50ac0ca143364042324046800c126a1d63816d532f2e19e6f2d8c0c31", size = 1619607 }, + { url = "https://files.pythonhosted.org/packages/f8/70/60bb08e9e9841b18d4669fb69d84b64ce900aacd7eb0ebebd4c7b9bdecd3/pycryptodomex-3.21.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0df2608682db8279a9ebbaf05a72f62a321433522ed0e499bc486a6889b96bf3", size = 1653571 }, + { url = "https://files.pythonhosted.org/packages/c9/6f/191b73509291c5ff0dddec9cc54797b1d73303c12b2e4017b24678e57099/pycryptodomex-3.21.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5823d03e904ea3e53aebd6799d6b8ec63b7675b5d2f4a4bd5e3adcb512d03b37", size = 1691548 }, + { url = "https://files.pythonhosted.org/packages/2d/c7/a0d3356f3074ac548afefa515ff46f3bea011deca607faf1c09b26dd5330/pycryptodomex-3.21.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:27e84eeff24250ffec32722334749ac2a57a5fd60332cd6a0680090e7c42877e", size = 1792099 }, + { url = "https://files.pythonhosted.org/packages/55/ee/9349856ee02826899fdc489016756865158217909a82dcc74cc4d55d33af/pycryptodomex-3.21.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:8ef436cdeea794015263853311f84c1ff0341b98fc7908e8a70595a68cefd971", size = 1619490 }, + { url = "https://files.pythonhosted.org/packages/07/93/e68fac121fcf761fd7a85a27f024c9238217e4d943c861a856ca354f412e/pycryptodomex-3.21.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a1058e6dfe827f4209c5cae466e67610bcd0d66f2f037465daa2a29d92d952b", size = 1653481 }, + { url = "https://files.pythonhosted.org/packages/b8/47/8a39243d09fd294c339c59834ba3c92715584f3ed0d92b6bacb26f803ce0/pycryptodomex-3.21.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9ba09a5b407cbb3bcb325221e346a140605714b5e880741dc9a1e9ecf1688d42", size = 1691442 }, + { url = "https://files.pythonhosted.org/packages/21/1c/f8860c558b44776573acd719c1e86fec14d42f29cf248eaba9c770151d14/pycryptodomex-3.21.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:8a9d8342cf22b74a746e3c6c9453cb0cfbb55943410e3a2619bd9164b48dc9d9", size = 1791966 }, +] + [[package]] name = "pydrive2" version = "1.20.0" @@ -1160,6 +1438,68 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/0c/0e3c05b1c87bb6a1c76d281b0f35e78d2d80ac91b5f8f524cebf77f51049/pyparsing-3.1.4-py3-none-any.whl", hash = "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c", size = 104100 }, ] +[[package]] +name = "pyppmd" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/39/c8/9039c7503577de08a3f4c81e7619583efdc16030da6d1a25268d3dca49c8/pyppmd-1.1.0.tar.gz", hash = "sha256:1d38ce2e4b7eb84b53bc8a52380b94f66ba6c39328b8800b30c2b5bf31693973", size = 1348949 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/10/b19621035862e2ae12a1ba14c5b5c0a0befb27906bc00691642d7bdbdce6/pyppmd-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c5cd428715413fe55abf79dc9fc54924ba7e518053e1fc0cbdf80d0d99cf1442", size = 75756 }, + { url = "https://files.pythonhosted.org/packages/85/4a/a7c172cd431c4e1ddf9be349dc4bcfea81c2a236d2fe51bbfdcd697af55a/pyppmd-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0e96cc43f44b7658be2ea764e7fa99c94cb89164dbb7cdf209178effc2168319", size = 47347 }, + { url = "https://files.pythonhosted.org/packages/0d/32/f7357e0412e977ede4d63ba8bf55d014e5ea5b311818b2b0a1fee6d91baa/pyppmd-1.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dd20142869094bceef5ab0b160f4fff790ad1f612313a1e3393a51fc3ba5d57e", size = 46640 }, + { url = "https://files.pythonhosted.org/packages/b5/8e/1f416819f0aab17de47b15b72d0e9b05e2bf795c6e28d9f403ac01398b74/pyppmd-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a4f9b51e45c11e805e74ea6f6355e98a6423b5bbd92f45aceee24761bdc3d3b8", size = 135666 }, + { url = "https://files.pythonhosted.org/packages/73/ac/7d07d3ac6874f235554de392de08e6a369001db43cd6a619af4fbe02fb55/pyppmd-1.1.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:459f85e928fb968d0e34fb6191fd8c4e710012d7d884fa2b317b2e11faac7c59", size = 132892 }, + { url = "https://files.pythonhosted.org/packages/09/76/61db4268a439cfba8736b14130d928d199633fab2360a2c5043332a427d2/pyppmd-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f73cf2aaf60477eef17f5497d14b6099d8be9748390ad2b83d1c88214d050c05", size = 138901 }, + { url = "https://files.pythonhosted.org/packages/8b/9c/546729489ae07c0d7c2bfe37c69ae1cd3ce35a18ab000480ea4e8f12754f/pyppmd-1.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2ea3ae0e92c0b5345cd3a4e145e01bbd79c2d95355481ea5d833b5c0cb202a2d", size = 139725 }, + { url = "https://files.pythonhosted.org/packages/f7/db/4e734e97541554a389e7adb2a2a5c86ad8ae35c4dafe817b12fdc317de1a/pyppmd-1.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:775172c740133c0162a01c1a5443d0e312246881cdd6834421b644d89a634b91", size = 131598 }, + { url = "https://files.pythonhosted.org/packages/b1/8f/530e47290e07d2fdedfd345fc72af08226ccdd4cc913c2b895a8396c17b6/pyppmd-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:14421030f1d46f69829698bdd960698a3b3df0925e3c470e82cfcdd4446b7bc1", size = 142767 }, + { url = "https://files.pythonhosted.org/packages/a5/f9/16e0adfef500b171a96ed3c95f4a4d999f99cc79de3e415146808b19c2fb/pyppmd-1.1.0-cp310-cp310-win32.whl", hash = "sha256:b691264f9962532aca3bba5be848b6370e596d0a2ca722c86df388be08d0568a", size = 41283 }, + { url = "https://files.pythonhosted.org/packages/37/8d/c4846ab632e13ead87189f31bcc51fc825c75078d162a4a9dc8aed0a5b97/pyppmd-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:216b0d969a3f06e35fbfef979706d987d105fcb1e37b0b1324f01ee143719c4a", size = 46078 }, + { url = "https://files.pythonhosted.org/packages/27/0e/9db5d7c6ca3159aa0f07c0f1d5c59079176e7c57740a61aca62a39661178/pyppmd-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1f8c51044ee4df1b004b10bf6b3c92f95ea86cfe1111210d303dca44a56e4282", size = 75781 }, + { url = "https://files.pythonhosted.org/packages/f0/1b/4894b5c71feee76d3dfccf4383b59841f9bfd27aecf912b6542a2ab1e073/pyppmd-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ac25b3a13d1ac9b8f0bde46952e10848adc79d932f2b548a6491ef8825ae0045", size = 47370 }, + { url = "https://files.pythonhosted.org/packages/50/98/57b2c281e546f682279bd4a2577045d1f6d527c8fa2151a990b2a9bc48c2/pyppmd-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c8d3003eebe6aabe22ba744a38a146ed58a25633420d5da882b049342b7c8036", size = 46633 }, + { url = "https://files.pythonhosted.org/packages/06/72/b7e37aa69b7a105bcc119bc171437fbcb104aef2568b68ec8ed21a3fcdd1/pyppmd-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c520656bc12100aa6388df27dd7ac738577f38bf43f4a4bea78e1861e579ea5", size = 138233 }, + { url = "https://files.pythonhosted.org/packages/60/73/4f53a3c7730e1cba3f210b35ed6779e0fe302739196f43452664e079c0b5/pyppmd-1.1.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c2a3e807028159a705951f5cb5d005f94caed11d0984e59cc50506de543e22d", size = 135486 }, + { url = "https://files.pythonhosted.org/packages/31/7c/956ebf1f07506bb59e6f13ef068d91f1bec828758d399b455b175b668f6c/pyppmd-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec8a2447e69444703e2b273247bfcd4b540ec601780eff07da16344c62d2993d", size = 141183 }, + { url = "https://files.pythonhosted.org/packages/73/b4/4863499e012c555f4619dbebc5b83d79818e0161d9b6fb8b1e709fb1d6c7/pyppmd-1.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b9e0c8053e69cad6a92a0889b3324f567afc75475b4f54727de553ac4fc85780", size = 141752 }, + { url = "https://files.pythonhosted.org/packages/b4/cc/44e175222b31f86d0192d1d0d2c46c4bf0e933c9a06a65ff39596ad05666/pyppmd-1.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:5938d256e8d2a2853dc3af8bb58ae6b4a775c46fc891dbe1826a0b3ceb624031", size = 133921 }, + { url = "https://files.pythonhosted.org/packages/f1/d9/2f2e222d43ab274909e8dcd16d25cd4cc0245a8d59f93f8d6397cd4dc49f/pyppmd-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1ce5822d8bea920856232ccfb3c26b56b28b6846ea1b0eb3d5cb9592a026649e", size = 145191 }, + { url = "https://files.pythonhosted.org/packages/6d/e7/1214571442624e2314ed1ed5ba0081358335fc760fb455c3d8df83b118c6/pyppmd-1.1.0-cp311-cp311-win32.whl", hash = "sha256:2a9e894750f2a52b03e3bc0d7cf004d96c3475a59b1af7e797d808d7d29c9ffe", size = 41286 }, + { url = "https://files.pythonhosted.org/packages/8e/7f/d3cc8443bd2b56bc54ea205dcf73d70ef8d4342096ff33fc8719956f45e9/pyppmd-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:969555c72e72fe2b4dd944127521a8f2211caddb5df452bbc2506b5adfac539e", size = 46087 }, + { url = "https://files.pythonhosted.org/packages/bf/0b/4c8e3a92c4366a9aa2d801ab4bd7ba72bd1d214da890dd91ab4d73e52878/pyppmd-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9d6ef8fd818884e914bc209f7961c9400a4da50d178bba25efcef89f09ec9169", size = 76116 }, + { url = "https://files.pythonhosted.org/packages/e1/0b/45fdf5a28c810ed4d3c0cb05ae5346e2972cdbfe89f374b263e07c5b820d/pyppmd-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:95f28e2ecf3a9656bd7e766aaa1162b6872b575627f18715f8b046e8617c124a", size = 47633 }, + { url = "https://files.pythonhosted.org/packages/56/a4/4aa1d36d98f3786c8b12ac96ac8234d7dc3c2a9e8f5174a5698f424099ec/pyppmd-1.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:37f3557ea65ee417abcdf5f49d35df00bb9f6f252639cae57aeefcd0dd596133", size = 46704 }, + { url = "https://files.pythonhosted.org/packages/d9/70/a49389a6666f670db5ecc7caa37030c9a9abfeea455c387172584551a271/pyppmd-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e84b25d088d7727d50218f57f92127cdb839acd6ec3de670b6680a4cf0b2d2a", size = 139145 }, + { url = "https://files.pythonhosted.org/packages/30/4c/f08cdf618744a3cce0da106ecf6e427b24d27b0bb1484afc40b88ca23a39/pyppmd-1.1.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:99ed42891986dac8c2ecf52bddfb777900233d867aa18849dbba6f3335600466", size = 136618 }, + { url = "https://files.pythonhosted.org/packages/bb/e0/afc0fb971c893e9e72cc8d70df93c50b3f3ebb12b4bdb21f869b775faf7e/pyppmd-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6fe69b82634488ada75ba07efb90cd5866fa3d64a2c12932b6e8ae207a14e5f", size = 142757 }, + { url = "https://files.pythonhosted.org/packages/26/b2/793e92c7a66de0b0b8d777c3c4df3ee5a5bec7fbaf0b69ab7374cefefa43/pyppmd-1.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:60981ffde1fe6ade750b690b35318c41a1160a8505597fda2c39a74409671217", size = 142749 }, + { url = "https://files.pythonhosted.org/packages/5e/6e/a1bf750bc7ed025a06600c65917d02e3c6dea7dfa728746c7251d4910d37/pyppmd-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:46e8240315476f57aac23d71e6de003e122b65feba7c68f4cc46a089a82a7cd4", size = 135033 }, + { url = "https://files.pythonhosted.org/packages/1e/ee/4a12a4b1990f1fabb77f9ef94d2cd6c795690eec79ad135b8236dc59dbd2/pyppmd-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c0308e2e76ecb4c878a18c2d7a7c61dbca89b4ef138f65d5f5ead139154dcdea", size = 146510 }, + { url = "https://files.pythonhosted.org/packages/04/cd/a6571420345315f5340ac10897726303ae07260cb025dc4a60371d1e8b97/pyppmd-1.1.0-cp312-cp312-win32.whl", hash = "sha256:b4fa4c27dc1314d019d921f2aa19e17f99250557e7569eeb70e180558f46af74", size = 41332 }, + { url = "https://files.pythonhosted.org/packages/c0/a4/af77129d671d6adcc6c82e1b0f03f0ad0b70c44ac70ed4c72b5c8952553b/pyppmd-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:c269d21e15f4175df27cf00296476097af76941f948734c642d7fb6e85b9b3b9", size = 46193 }, + { url = "https://files.pythonhosted.org/packages/b4/10/144f811290a36d6cf5b5c8ae9b68533abe4bea160285be73435d55c361e1/pyppmd-1.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2aeea1bf585c6b8771fa43a6abd704da92f8a46a6d0020953af15d7f3c82e48c", size = 75760 }, + { url = "https://files.pythonhosted.org/packages/dc/2b/242b6ba7938e77f14dadc0e5d638288b5588f0aff5e5d2c0428726606e5e/pyppmd-1.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7759bdb137694d4ab0cfa5ff2c75c212d90714c7da93544694f68001a0c38e12", size = 47339 }, + { url = "https://files.pythonhosted.org/packages/ec/80/91a13a5d0da916e7243f66839941976b2729bac1d3ca1737c20f1d59b216/pyppmd-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:db64a4fe956a2e700a737a1d019f526e6ccece217c163b28b354a43464cc495b", size = 46636 }, + { url = "https://files.pythonhosted.org/packages/aa/18/f6126af21186eee49f9aa090c36acc9d2bccef4c7d077d23b2f24dfb804c/pyppmd-1.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f788ae8f5a9e79cd777b7969d3401b2a2b87f47abe306c2a03baca30595e9bd", size = 135484 }, + { url = "https://files.pythonhosted.org/packages/03/55/ebfeb5d1085f8a2bd03f498aa7e6ef9635380bf1f88badd1f3b944198ada/pyppmd-1.1.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:324a178935c140210fca2043c688b77e79281da8172d2379a06e094f41735851", size = 132741 }, + { url = "https://files.pythonhosted.org/packages/df/92/f0a7a6e372c4bd659b5528ff179676522aa72bd8c7a071e757a490ff988e/pyppmd-1.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:363030bbcb7902fb9eeb59ffc262581ca5dd7790ba950328242fd2491c54d99b", size = 138761 }, + { url = "https://files.pythonhosted.org/packages/43/32/0f32a70ef3fbe287dc53fd86408e9c2f60515ab356bd728bde9fcebcb598/pyppmd-1.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:31b882584f86440b0ff7906385c9f9d9853e5799197abaafdae2245f87d03f01", size = 139568 }, + { url = "https://files.pythonhosted.org/packages/fd/02/730882ea61653af4ef22b3621e9a03e85e61c90884fb47df4495706439f5/pyppmd-1.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b991b4501492ec3380b605fe30bee0b61480d305e98519d81c2a658b2de01593", size = 131441 }, + { url = "https://files.pythonhosted.org/packages/9e/cd/624b6582766b8e60c4356ebcfe73504c81d1395f36d0d409ff081be56e3d/pyppmd-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b6108044d943b826f97a9e79201242f61392d6c1fadba463b2069c4e6bc961e1", size = 142558 }, + { url = "https://files.pythonhosted.org/packages/65/46/1908b1ef3ba6f1450bc1a3e45a6b39cfe4b33456a312d1add0041ba1bbe4/pyppmd-1.1.0-cp39-cp39-win32.whl", hash = "sha256:c45ce2968b7762d2cacf622b0a8f260295c6444e0883fd21a21017e3eaef16ed", size = 41284 }, + { url = "https://files.pythonhosted.org/packages/63/29/081b03f989deb7ce2f70461dac12ecc422e9abef6b7b7a1933945c96b06f/pyppmd-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:f5289f32ab4ec5f96a95da51309abd1769f928b0bff62047b3bc25c878c16ccb", size = 46071 }, + { url = "https://files.pythonhosted.org/packages/6a/e2/1d5fbd6dde1234b635000072c8d1d87c7ed3acf01a3c4aa8082504d58bc5/pyppmd-1.1.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ad5da9f7592158e6b6b51d7cd15e536d8b23afbb4d22cba4e5744c7e0a3548b1", size = 41505 }, + { url = "https://files.pythonhosted.org/packages/24/66/9215c5dda61b3aa3259902a586dacd198b4b0793ab99228734091b5e7fa7/pyppmd-1.1.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc6543e7d12ef0a1466d291d655e3d6bca59c7336dbb53b62ccdd407822fb52b", size = 44814 }, + { url = "https://files.pythonhosted.org/packages/1a/87/cc2aa429688f238ae30f26b8334194a21e25643d3257c9e5b14cccdc578e/pyppmd-1.1.0-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a5e4008a45910e3c8c227f6f240de67eb14454c015dc3d8060fc41e230f395d3", size = 43629 }, + { url = "https://files.pythonhosted.org/packages/9f/96/cd3f64f6bdce091ffb6d2c1c23dc91e8b94e312a5d08cd648625555fb69e/pyppmd-1.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9301fa39d1fb0ed09a10b4c5d7f0074113e96a1ead16ba7310bedf95f7ef660c", size = 43911 }, + { url = "https://files.pythonhosted.org/packages/e6/ab/02ab90e2dddf2dd55e30e64fa0509627c6e0c86b26503a6df95ae55b1e45/pyppmd-1.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:59521a3c6028da0cb5780ba16880047b00163432a6b975da2f6123adfc1b0be8", size = 42427 }, + { url = "https://files.pythonhosted.org/packages/fa/0e/05db05c0da6a9bbb1f32de107b9f92e95ca9cb407c2082c7a0bee0a8868b/pyppmd-1.1.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:cce8cd2d4ceebe2dbf41db6dfebe4c2e621314b3af8a2df2cba5eb5fa277f122", size = 41500 }, + { url = "https://files.pythonhosted.org/packages/ba/89/48b01bcab274c59548897de336b807777b8e4abbd0465ed37c04152d13e2/pyppmd-1.1.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62e57927dbcb91fb6290a41cd83743b91b9d85858efb16a0dd34fac208ee1c6b", size = 44810 }, + { url = "https://files.pythonhosted.org/packages/a9/e4/bf4aa2305eda1bdf530d8127e07dd324ae923e664db3aa7bf153f1d749ec/pyppmd-1.1.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:435317949a6f35e54cdf08e0af6916ace427351e7664ac1593980114668f0aaa", size = 43625 }, + { url = "https://files.pythonhosted.org/packages/fc/be/33478c13fad90049c35034d9f7de658353f82fc2cd91c57efd904b71ebb7/pyppmd-1.1.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f66b0d0e32b8fb8707f1d2552f13edfc2917e8ed0bdf4d62e2ce190d2c70834", size = 43909 }, + { url = "https://files.pythonhosted.org/packages/60/34/922d8ca6879f08d17f9771e6ef65e9491c2ebbd48934997a3ff01285e55e/pyppmd-1.1.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:650a663a591e06fb8096c213f4070b158981c8c3bf9c166ce7e4c360873f2750", size = 42422 }, +] + [[package]] name = "pyright" version = "1.1.373" @@ -1285,6 +1625,96 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/87/5124b1c1f2412bb95c59ec481eaf936cd32f0fe2a7b16b97b81c4c017a6a/PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8", size = 162312 }, ] +[[package]] +name = "pyzstd" +version = "0.16.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/62/14/878fee4072cecb1cc6e061c7d0d933e481389c27de939538c9cc3f18894a/pyzstd-0.16.2.tar.gz", hash = "sha256:179c1a2ea1565abf09c5f2fd72f9ce7c54b2764cf7369e05c0bfd8f1f67f63d2", size = 789505 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/a9/efad061c5a982f859ba8bf5de565d73567f87ad8bba3364fe28e9a8672b6/pyzstd-0.16.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:637376c8f8cbd0afe1cab613f8c75fd502bd1016bf79d10760a2d5a00905fe62", size = 372191 }, + { url = "https://files.pythonhosted.org/packages/b6/36/eb6dcfacb273ca13dfa20d296f27ffd0a6c53677965f868625edf764b71e/pyzstd-0.16.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3e7a7118cbcfa90ca2ddbf9890c7cb582052a9a8cf2b7e2c1bbaf544bee0f16a", size = 295083 }, + { url = "https://files.pythonhosted.org/packages/fb/76/a7862487402123f221439808ed50915e00cfc8e1df7365af366610176347/pyzstd-0.16.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a74cb1ba05876179525144511eed3bd5a509b0ab2b10632c1215a85db0834dfd", size = 390166 }, + { url = "https://files.pythonhosted.org/packages/b8/52/1e1ab63026d67f18b9841285576d59bb799b838a5de4f852ad9e054674a1/pyzstd-0.16.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c084dde218ffbf112e507e72cbf626b8f58ce9eb23eec129809e31037984662", size = 472043 }, + { url = "https://files.pythonhosted.org/packages/0d/24/14c8948b9d16d399ff80504bc404bb091b0eb5339f6fbdad0481da751c09/pyzstd-0.16.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4646459ebd3d7a59ddbe9312f020bcf7cdd1f059a2ea07051258f7af87a0b31", size = 415258 }, + { url = "https://files.pythonhosted.org/packages/6b/3e/e4c7f449af9d19975ff5d333a58330317cf8b05fe4754106c694a29e7c25/pyzstd-0.16.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14bfc2833cc16d7657fc93259edeeaa793286e5031b86ca5dc861ba49b435fce", size = 413680 }, + { url = "https://files.pythonhosted.org/packages/10/09/8918853028cf593c141456b9a42d68420beec3f16a8cc4f1aa5d0b8b0c84/pyzstd-0.16.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f27d488f19e5bf27d1e8aa1ae72c6c0a910f1e1ffbdf3c763d02ab781295dd27", size = 412630 }, + { url = "https://files.pythonhosted.org/packages/47/20/5a4c899530571e0e8ecdcb9dc7e3fc38491d4b342fbd7d8413805c88013b/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:91e134ca968ff7dcfa8b7d433318f01d309b74ee87e0d2bcadc117c08e1c80db", size = 404980 }, + { url = "https://files.pythonhosted.org/packages/0a/1d/aeeeebb702d3500a01b5b1029ba1716aea3afa75e8aacb904806b3f1afe5/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:6b5f64cd3963c58b8f886eb6139bb8d164b42a74f8a1bb95d49b4804f4592d61", size = 418000 }, + { url = "https://files.pythonhosted.org/packages/fc/0c/66ca36d24ad97af40a8fe8de9e3f316a5f4fd2fb3cab8634a2f7da5571c8/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:0b4a8266871b9e0407f9fd8e8d077c3558cf124d174e6357b523d14f76971009", size = 485576 }, + { url = "https://files.pythonhosted.org/packages/39/66/6c1de1347de94aa85f60e854cccae0948bda2eda2351e4d47c8bb0a7cf18/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:1bb19f7acac30727354c25125922aa59f44d82e0e6a751df17d0d93ff6a73853", size = 564542 }, + { url = "https://files.pythonhosted.org/packages/6d/46/75365a3ab279d58e69d410ce0a21527e689fa651837227e23dee294d096f/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3008325b7368e794d66d4d98f2ee1d867ef5afd09fd388646ae02b25343c420d", size = 430619 }, + { url = "https://files.pythonhosted.org/packages/0d/62/17bf81d42acbd39bffdea559b6fbd7ec331cd74bc52f249e536fefe5480d/pyzstd-0.16.2-cp310-cp310-win32.whl", hash = "sha256:66f2d5c0bbf5bf32c577aa006197b3525b80b59804450e2c32fbcc2d16e850fd", size = 218224 }, + { url = "https://files.pythonhosted.org/packages/f7/b6/281245890df08a567186c6e262c43d68581291cca107c8d7304c37708e46/pyzstd-0.16.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fe5f5459ebe1161095baa7a86d04ab625b35148f6c425df0347ed6c90a2fd58", size = 245012 }, + { url = "https://files.pythonhosted.org/packages/10/5a/19d7aec81853f6dc53eabad388227e3beecfaca4788af23b8807a0ea2112/pyzstd-0.16.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1bdbe7f01c7f37d5cd07be70e32a84010d7dfd6677920c0de04cf7d245b60d", size = 372192 }, + { url = "https://files.pythonhosted.org/packages/29/35/2eb025e6a0fff49b5de8bea20e82e4d7d5456e634bf3809123fbe5e5f194/pyzstd-0.16.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1882a3ceaaf9adc12212d587d150ec5e58cfa9a765463d803d739abbd3ac0f7a", size = 295084 }, + { url = "https://files.pythonhosted.org/packages/04/1f/03785d7ff1ce73b9347533f798cb27afa57768e66012f97b18b7b7303158/pyzstd-0.16.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea46a8b9d60f6a6eba29facba54c0f0d70328586f7ef0da6f57edf7e43db0303", size = 390167 }, + { url = "https://files.pythonhosted.org/packages/b7/59/e307622115a2df30075efbd28933dc0ad8f2007c5ba5a3eb49c956de3d56/pyzstd-0.16.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d7865bc06589cdcecdede0deefe3da07809d5b7ad9044c224d7b2a0867256957", size = 472038 }, + { url = "https://files.pythonhosted.org/packages/97/21/870fda5454240089e9c37625320580d392b03beaeae4889c67c0a21c4d34/pyzstd-0.16.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:52f938a65b409c02eb825e8c77fc5ea54508b8fc44b5ce226db03011691ae8cc", size = 415217 }, + { url = "https://files.pythonhosted.org/packages/3c/35/b33faeeb9c96fddd08bf7871c9f5c4638c32ad79227155922fd4a63190c5/pyzstd-0.16.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e97620d3f53a0282947304189deef7ca7f7d0d6dfe15033469dc1c33e779d5e5", size = 413714 }, + { url = "https://files.pythonhosted.org/packages/aa/a3/b9058dd43eb52025a2ca78946dcb9ef9d8984acac172a698bcf12712217c/pyzstd-0.16.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7c40e9983d017108670dc8df68ceef14c7c1cf2d19239213274783041d0e64c", size = 412568 }, + { url = "https://files.pythonhosted.org/packages/12/31/fe7d462c912f2040775bfa2af4327f9fcebb16e8fa9c3bfa058bc1306722/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7cd4b3b2c6161066e4bde6af1cf78ed3acf5d731884dd13fdf31f1db10830080", size = 404988 }, + { url = "https://files.pythonhosted.org/packages/48/4c/582aca0e5210436499bce1639a8d15da3f76f8d5827da1aa3eeb2c4e271c/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:454f31fd84175bb203c8c424f2255a343fa9bd103461a38d1bf50487c3b89508", size = 417961 }, + { url = "https://files.pythonhosted.org/packages/39/e9/54f53641ff10b4ea18d3ba159b03bd07e6ae5a5b7ae01f1329b0c35b8ca2/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:5ef754a93743f08fb0386ce3596780bfba829311b49c8f4107af1a4bcc16935d", size = 485587 }, + { url = "https://files.pythonhosted.org/packages/ce/65/25243b3fea9e52a20bfece1b12e3d3ee3125f17b1735aab08cb9a7a760b4/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:be81081db9166e10846934f0e3576a263cbe18d81eca06e6a5c23533f8ce0dc6", size = 564543 }, + { url = "https://files.pythonhosted.org/packages/3b/3c/324b8ddca55b4b073b413cea3e0587af3c8153ccf7d6d63ed294831f2095/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:738bcb2fa1e5f1868986f5030955e64de53157fa1141d01f3a4daf07a1aaf644", size = 430628 }, + { url = "https://files.pythonhosted.org/packages/db/a1/aca18925e23bceb833fc742ebaf87aa9d1ba8b178f0332bd108fc8966482/pyzstd-0.16.2-cp311-cp311-win32.whl", hash = "sha256:0ea214c9b97046867d1657d55979021028d583704b30c481a9c165191b08d707", size = 218215 }, + { url = "https://files.pythonhosted.org/packages/c0/7f/0f5d1d1891e6c6e14d846d2881a06ab7e5e97cabeb5e1e9e53debec4091a/pyzstd-0.16.2-cp311-cp311-win_amd64.whl", hash = "sha256:c17c0fc02f0e75b0c7cd21f8eaf4c6ce4112333b447d93da1773a5f705b2c178", size = 245055 }, + { url = "https://files.pythonhosted.org/packages/28/15/20046759d138733e7150afa6aa15f322022d7587968e2dbd5b36fbf8aa86/pyzstd-0.16.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d4081fd841a9efe9ded7290ee7502dbf042c4158b90edfadea3b8a072c8ec4e1", size = 373230 }, + { url = "https://files.pythonhosted.org/packages/51/8d/55b536edaecf19d2f8dbd8fbaefd184f2f9cc6b71d241caa6d86bed96813/pyzstd-0.16.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fd3fa45d2aeb65367dd702806b2e779d13f1a3fa2d13d5ec777cfd09de6822de", size = 295699 }, + { url = "https://files.pythonhosted.org/packages/11/14/086e7f690154c6f3d9bdb46da26a4cd3c9e0b284346ce10943711ca48c32/pyzstd-0.16.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8b5f0d2c07994a5180d8259d51df6227a57098774bb0618423d7eb4a7303467", size = 390556 }, + { url = "https://files.pythonhosted.org/packages/90/d2/c6d854705d6fa0ad876209b4ba796ab31d85b710d1459029f2cb41085a8d/pyzstd-0.16.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60c9d25b15c7ae06ed5d516d096a0d8254f9bed4368b370a09cccf191eaab5cb", size = 472928 }, + { url = "https://files.pythonhosted.org/packages/aa/38/f97dd871e446adc834349caa605dbaf5bac86763a255f62c809cc2459c85/pyzstd-0.16.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:29acf31ce37254f6cad08deb24b9d9ba954f426fa08f8fae4ab4fdc51a03f4ae", size = 416057 }, + { url = "https://files.pythonhosted.org/packages/53/be/0c5ad7bf29dc890f6a3303760b9802aeeafa4e3ffb598de625f501986bfe/pyzstd-0.16.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ec77612a17697a9f7cf6634ffcee616eba9b997712fdd896e77fd19ab3a0618", size = 414613 }, + { url = "https://files.pythonhosted.org/packages/1f/1a/d3a1edcd59e2f62a35ac6257d2b86a2c872ae9a8e925380620a8db0d9a9a/pyzstd-0.16.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:313ea4974be93be12c9a640ab40f0fc50a023178aae004a8901507b74f190173", size = 413236 }, + { url = "https://files.pythonhosted.org/packages/f2/8d/912430c2310466c14a89a5a529b72eddef7e73fa733806dbe0b030cf3495/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e91acdefc8c2c6c3b8d5b1b5fe837dce4e591ecb7c0a2a50186f552e57d11203", size = 405536 }, + { url = "https://files.pythonhosted.org/packages/9e/83/4edb419a13b9d1e1debc01e88084eba93a5f7c10ef198da11f6782857c73/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:929bd91a403539e72b5b5cb97f725ac4acafe692ccf52f075e20cd9bf6e5493d", size = 419145 }, + { url = "https://files.pythonhosted.org/packages/8f/e9/62a169eddc37aefac480ee3b3318c221f6731e1e342dafd9e05b7fdaa7c5/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:740837a379aa32d110911ebcbbc524f9a9b145355737527543a884bd8777ca4f", size = 487157 }, + { url = "https://files.pythonhosted.org/packages/57/9d/5949f2a0144d1f99fab7914f854b582d2784c73139cc190e603e4d6b7b37/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:adfc0e80dd157e6d1e0b0112c8ecc4b58a7a23760bd9623d74122ef637cfbdb6", size = 565918 }, + { url = "https://files.pythonhosted.org/packages/de/ce/647b9c7602ac477c9e62cf9399810f72bb5dba8f508e7cdf8be1d260e6f9/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:79b183beae1c080ad3dca39019e49b7785391947f9aab68893ad85d27828c6e7", size = 431373 }, + { url = "https://files.pythonhosted.org/packages/8b/fb/4141e3d4549eea26e5a59ec723eade271980816cb2ed7613df855baa672f/pyzstd-0.16.2-cp312-cp312-win32.whl", hash = "sha256:b8d00631a3c466bc313847fab2a01f6b73b3165de0886fb03210e08567ae3a89", size = 218541 }, + { url = "https://files.pythonhosted.org/packages/51/b9/e1373b179129c2095d70bd1df02a51d388f4c7e4ecb62acb4e5e9570269b/pyzstd-0.16.2-cp312-cp312-win_amd64.whl", hash = "sha256:c0d43764e9a60607f35d8cb3e60df772a678935ab0e02e2804d4147377f4942c", size = 245320 }, + { url = "https://files.pythonhosted.org/packages/66/10/cc7c764c7673f1af1728abdcf58e58f88ef5d44ab4500677a2b7b4c01e7d/pyzstd-0.16.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3ae9ae7ad730562810912d7ecaf1fff5eaf4c726f4b4dfe04784ed5f06d7b91f", size = 373223 }, + { url = "https://files.pythonhosted.org/packages/3f/a7/bcaf7d635ee929dd4d08ae1c35101892db56a11542471eecfbf46b9dd988/pyzstd-0.16.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2ce8d3c213f76a564420f3d0137066ac007ce9fb4e156b989835caef12b367a7", size = 295701 }, + { url = "https://files.pythonhosted.org/packages/93/49/a604113a2f3135b29371a894c0faad22d7ea3f7b58f38d77baad8a817483/pyzstd-0.16.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c2c14dac23c865e2d78cebd9087e148674b7154f633afd4709b4cd1520b99a61", size = 392395 }, + { url = "https://files.pythonhosted.org/packages/b0/38/886ecf3ebb13a4b6e3ee85f448f54eef37a5ae2b453bd9d5d9edc909e119/pyzstd-0.16.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4527969d66a943e36ef374eda847e918077de032d58b5df84d98ffd717b6fa77", size = 474523 }, + { url = "https://files.pythonhosted.org/packages/14/98/121da6ac072c00090c218b4888ef00ead15979f09a657d9a5ff770d6bb17/pyzstd-0.16.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd8256149b88e657e99f31e6d4b114c8ff2935951f1d8bb8e1fe501b224999c0", size = 417974 }, + { url = "https://files.pythonhosted.org/packages/b6/ba/56652a67c0bcfaceb2945e5f07d5aa21af86e07cf33d1ae47bb3529a56c3/pyzstd-0.16.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5bd1f1822d65c9054bf36d35307bf8ed4aa2d2d6827431761a813628ff671b1d", size = 414587 }, + { url = "https://files.pythonhosted.org/packages/cc/30/cab6f45101f0113ced609ef65482aedd276e0f022d9f25a327d4284142f5/pyzstd-0.16.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6733f4d373ec9ad2c1976cf06f973a3324c1f9abe236d114d6bb91165a397d", size = 415071 }, + { url = "https://files.pythonhosted.org/packages/6d/44/2187fc8a46662926943aeb16d639dd4f3d06267c7e8abb2c6f97700ab11c/pyzstd-0.16.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7bec165ab6524663f00b69bfefd13a46a69fed3015754abaf81b103ec73d92c6", size = 407835 }, + { url = "https://files.pythonhosted.org/packages/de/d5/6edca97d5453cba820d2ad5630e6ec1fcfad66f69af5ad7d6c688ea301be/pyzstd-0.16.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e4460fa6949aac6528a1ad0de8871079600b12b3ef4db49316306786a3598321", size = 421755 }, + { url = "https://files.pythonhosted.org/packages/54/c1/1a0339e014ed97f4e6fd9166b0409ceda8f32e28e8ecda70fd7bb0915566/pyzstd-0.16.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:75df79ea0315c97d88337953a17daa44023dbf6389f8151903d371513f503e3c", size = 489174 }, + { url = "https://files.pythonhosted.org/packages/07/01/c65f2c9f0b902b33efcb0bdf3cbd07fc828fda6ff6333189eb71cf7acc60/pyzstd-0.16.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:93e1d45f4a196afb6f18682c79bdd5399277ead105b67f30b35c04c207966071", size = 573025 }, + { url = "https://files.pythonhosted.org/packages/a7/54/7ab9cc54171b7f8bb97cfd1c1aa7fcb706a4babeb629732529d8111bc4e6/pyzstd-0.16.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:075e18b871f38a503b5d23e40a661adfc750bd4bd0bb8b208c1e290f3ceb8fa2", size = 429582 }, + { url = "https://files.pythonhosted.org/packages/6c/a5/f9c950bb378dd1335bc4cc56444ec2ab40b1dab085c5798c5d16a9bf9d0b/pyzstd-0.16.2-cp313-cp313-win32.whl", hash = "sha256:9e4295eb299f8d87e3487852bca033d30332033272a801ca8130e934475e07a9", size = 218544 }, + { url = "https://files.pythonhosted.org/packages/9a/df/a15b9a8a59cd9908ae2b70bce2cb4ac3e2d7da11414ee0d0ceb46e4d0439/pyzstd-0.16.2-cp313-cp313-win_amd64.whl", hash = "sha256:18deedc70f858f4cf574e59f305d2a0678e54db2751a33dba9f481f91bc71c28", size = 245313 }, + { url = "https://files.pythonhosted.org/packages/e0/38/43002103a545bc953e532973596e905550e9626973c1b282e04e01038ac6/pyzstd-0.16.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a9892b707ef52f599098b1e9528df0e7849c5ec01d3e8035fb0e67de4b464839", size = 372192 }, + { url = "https://files.pythonhosted.org/packages/61/be/28dfeba9dbad8ed19d6aefa0d6623d1ee97e83c6c1e97910439428655f28/pyzstd-0.16.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4fbd647864341f3c174c4a6d7f20e6ea6b4be9d840fb900dc0faf0849561badc", size = 295080 }, + { url = "https://files.pythonhosted.org/packages/63/c2/c7e5244f2dde72df3fb2b7b952e8d01bac20cd78dc0d585d0a060ca565b0/pyzstd-0.16.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20ac2c15656cc6194c4fed1cb0e8159f9394d4ea1d58be755448743d2ec6c9c4", size = 390165 }, + { url = "https://files.pythonhosted.org/packages/ff/30/52560cb88179fa3ff7536429c0d7b83aeecea86ecb2d180a4afc991502e5/pyzstd-0.16.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b239fb9a20c1be3374b9a2bd183ba624fd22ad7a3f67738c0d80cda68b4ae1d3", size = 472040 }, + { url = "https://files.pythonhosted.org/packages/69/a7/ab1e19626da5a8ff58493d6928d9d0da4931034e7a124949bf1a1705daaf/pyzstd-0.16.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cc52400412cdae2635e0978b8d6bcc0028cc638fdab2fd301f6d157675d26896", size = 415255 }, + { url = "https://files.pythonhosted.org/packages/28/0d/bf7c9388fe43c7051a2ced4645e58a493a35c62e68307b5aaf0fb129b008/pyzstd-0.16.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b766a6aeb8dbb6c46e622e7a1aebfa9ab03838528273796941005a5ce7257b1", size = 413679 }, + { url = "https://files.pythonhosted.org/packages/58/2a/1e0738740a8bd2b1f4a74be86297c5776936b66b3a5340d8e4ae84c5844f/pyzstd-0.16.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd4b8676052f9d59579242bf3cfe5fd02532b6a9a93ab7737c118ae3b8509dc", size = 412623 }, + { url = "https://files.pythonhosted.org/packages/23/d5/7cbfbebbb3ffccb0626fc2fab622fb5a10cf66c2c60481f51e46a92eb2c5/pyzstd-0.16.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1c6c0a677aac7c0e3d2d2605d4d68ffa9893fdeeb2e071040eb7c8750969d463", size = 404981 }, + { url = "https://files.pythonhosted.org/packages/a7/b0/6ac198c753cc135357630e856f40f5998c2d28609713ae2830c679e8248c/pyzstd-0.16.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:15f9c2d612e7e2023d68d321d1b479846751f792af89141931d44e82ae391394", size = 417997 }, + { url = "https://files.pythonhosted.org/packages/c6/8f/0e5685efbf24ae62e135549e37947ca7919616b81108584112e25dd1a55a/pyzstd-0.16.2-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:11740bff847aad23beef4085a1bb767d101895881fe891f0a911aa27d43c372c", size = 485576 }, + { url = "https://files.pythonhosted.org/packages/30/d6/bf2f05752082967ac748d7c2d7c5a71097ac6fc1b902b5d34764cd0c12f7/pyzstd-0.16.2-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:b9067483ebe860e4130a03ee665b3d7be4ec1608b208e645d5e7eb3492379464", size = 564538 }, + { url = "https://files.pythonhosted.org/packages/d8/97/1081cc3cbf5eeb6cf4e385226e9989fdebb61f8e48baa210eb774145e667/pyzstd-0.16.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:988f0ba19b14c2fe0afefc444ac1edfb2f497b7d7c3212b2f587504cc2ec804e", size = 430615 }, + { url = "https://files.pythonhosted.org/packages/e0/a7/2a82fbb248b951434306dd77e969fb99305968904c9a7494574d696b1392/pyzstd-0.16.2-cp39-cp39-win32.whl", hash = "sha256:8855acb1c3e3829030b9e9e9973b19e2d70f33efb14ad5c474b4d086864c959c", size = 218215 }, + { url = "https://files.pythonhosted.org/packages/9d/bf/e529ff84b87c8f978ab35906921ac54841270562e65bcb5d0dd9d3240204/pyzstd-0.16.2-cp39-cp39-win_amd64.whl", hash = "sha256:018e88378df5e76f5e1d8cf4416576603b6bc4a103cbc66bb593eaac54c758de", size = 245047 }, + { url = "https://files.pythonhosted.org/packages/f9/ad/c09fb722c12a82b826c97efc50a919e229bfbaf644f5a140adcd71941473/pyzstd-0.16.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4b631117b97a42ff6dfd0ffc885a92fff462d7c34766b28383c57b996f863338", size = 364187 }, + { url = "https://files.pythonhosted.org/packages/57/f9/93175fe72f85fb675fe04abca296fe583112a25d0ec7faa026288d9463c2/pyzstd-0.16.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:56493a3fbe1b651a02102dd0902b0aa2377a732ff3544fb6fb3f114ca18db52f", size = 279825 }, + { url = "https://files.pythonhosted.org/packages/8a/de/0b40acf76d7ed1f7975877535e004de85ec2e869632754b5d4d389258b8a/pyzstd-0.16.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1eae9bdba4a1e5d3181331f403114ff5b8ce0f4b569f48eba2b9beb2deef1e4", size = 321313 }, + { url = "https://files.pythonhosted.org/packages/41/5e/00102bacd1a7c957c88098f3ae2cdac17842ac0f94d2e685ff5b75a05730/pyzstd-0.16.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1be6972391c8aeecc7e61feb96ffc8e77a401bcba6ed994e7171330c45a1948", size = 344376 }, + { url = "https://files.pythonhosted.org/packages/a3/95/27a7da3dbd4460cd9432bdc22d9d5f8ec77c86275d069020fa74ea280f7f/pyzstd-0.16.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:761439d687e3a5687c2ff5c6a1190e1601362a4a3e8c6c82ff89719d51d73e19", size = 328591 }, + { url = "https://files.pythonhosted.org/packages/c2/03/8f4d5fd45f6bfad66d67cdf583492a9f52a21049f60e6b36a7e9f8aa7adc/pyzstd-0.16.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f5fbdb8cf31b60b2dc586fecb9b73e2f172c21a0b320ed275f7b8d8a866d9003", size = 240786 }, + { url = "https://files.pythonhosted.org/packages/91/f6/bd63e2587e0ec40abd9f92278a442bc28b7ff109e418d1240ee2eb6536aa/pyzstd-0.16.2-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:183f26e34f9becf0f2db38be9c0bfb136753d228bcb47c06c69175901bea7776", size = 364180 }, + { url = "https://files.pythonhosted.org/packages/ac/13/d4c68ad926e79d734f57b26d49447908e8dab7f5c066d3a013b0d0cfa2be/pyzstd-0.16.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:88318b64b5205a67748148d6d244097fa6cf61fcea02ad3435511b9e7155ae16", size = 279816 }, + { url = "https://files.pythonhosted.org/packages/b2/ba/76f0b75ec9e9fc3914496e036f99f345d5e0a99cb7070341f9becdaba2b8/pyzstd-0.16.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73142aa2571b6480136a1865ebda8257e09eabbc8bcd54b222202f6fa4febe1e", size = 321308 }, + { url = "https://files.pythonhosted.org/packages/a6/ea/9fe52bd777f33f007287f1a37bada7af5cf33d64904360c17bb64fefca21/pyzstd-0.16.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d3f8877c29a97f1b1bba16f3d3ab01ad10ad3da7bad317aecf36aaf8848b37c", size = 344368 }, + { url = "https://files.pythonhosted.org/packages/cc/c0/509077f73fc8e156ceeefb41d4b7e04aceb71b2339084fcd62d0ad3bfd75/pyzstd-0.16.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1f25754562473ac7de856b8331ebd5964f5d85601045627a5f0bb0e4e899990", size = 328585 }, + { url = "https://files.pythonhosted.org/packages/14/74/a854ada61bf4c3c2ad239ec2bd1ff73cc0d718ccbcc56e3ced94e878fd50/pyzstd-0.16.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:6ce17e84310080c55c02827ad9bb17893c00a845c8386a328b346f814aabd2c1", size = 240783 }, +] + [[package]] name = "rdata" version = "0.9" @@ -1443,26 +1873,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.1.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/25/4c/2f786388acd82c295eedc4afeede7ef4b29cf27277151d8d13be906bac70/ruff-0.1.6.tar.gz", hash = "sha256:1b09f29b16c6ead5ea6b097ef2764b42372aebe363722f1605ecbcd2b9207184", size = 1719627 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/75/5054ec93ec0d5db26e218cb2814ddaa085ba1f29fad0ec56dd8107a97688/ruff-0.1.6-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:88b8cdf6abf98130991cbc9f6438f35f6e8d41a02622cc5ee130a02a0ed28703", size = 11628468 }, - { url = "https://files.pythonhosted.org/packages/a2/91/8b2920f6026c069ae0802fc3c44f7337e04bf2a198ce94bfab360073477a/ruff-0.1.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5c549ed437680b6105a1299d2cd30e4964211606eeb48a0ff7a93ef70b902248", size = 5990066 }, - { url = "https://files.pythonhosted.org/packages/df/1e/03ef0cc5c7d03e50d4f954218551d6001f1f70e6f391cdb678efb5c6e6ab/ruff-0.1.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cf5f701062e294f2167e66d11b092bba7af6a057668ed618a9253e1e90cfd76", size = 5929313 }, - { url = "https://files.pythonhosted.org/packages/92/7c/38fd1b9cb624f5725a6a08c81bf7e823c64b28622ffcb4369c56dc0a16d0/ruff-0.1.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:05991ee20d4ac4bb78385360c684e4b417edd971030ab12a4fbd075ff535050e", size = 5627943 }, - { url = "https://files.pythonhosted.org/packages/c7/c3/98e3d0eb92e5a2ec10f76c71067640b6f21def23c3b1ff8f08ab6348255e/ruff-0.1.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87455a0c1f739b3c069e2f4c43b66479a54dea0276dd5d4d67b091265f6fd1dc", size = 6074206 }, - { url = "https://files.pythonhosted.org/packages/e8/33/62fb966eb70d9bb45ddf5023d40e26946a5e5127d99956b84c8a9a76b153/ruff-0.1.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:683aa5bdda5a48cb8266fcde8eea2a6af4e5700a392c56ea5fb5f0d4bfdc0240", size = 6743205 }, - { url = "https://files.pythonhosted.org/packages/c7/f1/60d43182f98113156a1b21a17f30541dda9f5ffcfeedc2b54dc030a2c413/ruff-0.1.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:137852105586dcbf80c1717facb6781555c4e99f520c9c827bd414fac67ddfb6", size = 6600581 }, - { url = "https://files.pythonhosted.org/packages/09/92/36850598e84f75cfe8edd252dbf40442b4cc226ed2c76206a9b3cbfb9986/ruff-0.1.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd98138a98d48a1c36c394fd6b84cd943ac92a08278aa8ac8c0fdefcf7138f35", size = 7708098 }, - { url = "https://files.pythonhosted.org/packages/3b/2f/8ef67614631622aa3ea79b27e01ac86d7f90a988520454e3a84cb2fd890f/ruff-0.1.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a0cd909d25f227ac5c36d4e7e681577275fb74ba3b11d288aff7ec47e3ae745", size = 6295145 }, - { url = "https://files.pythonhosted.org/packages/3c/4b/af366db98d15efe83fd3e3aae7319d3897e3475fc53a2f1b0287c8255422/ruff-0.1.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e8fd1c62a47aa88a02707b5dd20c5ff20d035d634aa74826b42a1da77861b5ff", size = 5878111 }, - { url = "https://files.pythonhosted.org/packages/bf/af/25b794e750f1d74a83ce6b16625e3306beeb2161c517b9d883958de05526/ruff-0.1.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fd89b45d374935829134a082617954120d7a1470a9f0ec0e7f3ead983edc48cc", size = 5629339 }, - { url = "https://files.pythonhosted.org/packages/81/b0/92c4cb6bceb19ebd27cedd1f45b337f7fd5397e6b760094831266be59661/ruff-0.1.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:491262006e92f825b145cd1e52948073c56560243b55fb3b4ecb142f6f0e9543", size = 5959772 }, - { url = "https://files.pythonhosted.org/packages/11/02/3a7e3101d88b113f326e0fdf3f566fba2600fc4b1fd828d56027d293e22d/ruff-0.1.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ea284789861b8b5ca9d5443591a92a397ac183d4351882ab52f6296b4fdd5462", size = 6356759 }, - { url = "https://files.pythonhosted.org/packages/fc/93/8a4b3523c4564168aab720d8361b4bf12173b099509caaac93168b72927a/ruff-0.1.6-py3-none-win32.whl", hash = "sha256:1610e14750826dfc207ccbcdd7331b6bd285607d4181df9c1c6ae26646d6848a", size = 5835632 }, - { url = "https://files.pythonhosted.org/packages/40/e3/55ae013087bd892dd61c25895a119ffec2f9b9f7c3d1fd0cea5f5f7bd74a/ruff-0.1.6-py3-none-win_amd64.whl", hash = "sha256:4558b3e178145491e9bc3b2ee3c4b42f19d19384eaa5c59d10acf6e8f8b57e33", size = 6226075 }, - { url = "https://files.pythonhosted.org/packages/21/f7/fdce733e594e1b274fc9232256be6a4e03e8d2cb9e354c783801191e2f4d/ruff-0.1.6-py3-none-win_arm64.whl", hash = "sha256:03910e81df0d8db0e30050725a5802441c2022ea3ae4fe0609b76081731accbc", size = 5965823 }, +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/2b/01245f4f3a727d60bebeacd7ee6d22586c7f62380a2597ddb22c2f45d018/ruff-0.8.2.tar.gz", hash = "sha256:b84f4f414dda8ac7f75075c1fa0b905ac0ff25361f42e6d5da681a465e0f78e5", size = 3349020 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/29/366be70216dba1731a00a41f2f030822b0c96c7c4f3b2c0cdce15cbace74/ruff-0.8.2-py3-none-linux_armv6l.whl", hash = "sha256:c49ab4da37e7c457105aadfd2725e24305ff9bc908487a9bf8d548c6dad8bb3d", size = 10530649 }, + { url = "https://files.pythonhosted.org/packages/63/82/a733956540bb388f00df5a3e6a02467b16c0e529132625fe44ce4c5fb9c7/ruff-0.8.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ec016beb69ac16be416c435828be702ee694c0d722505f9c1f35e1b9c0cc1bf5", size = 10274069 }, + { url = "https://files.pythonhosted.org/packages/3d/12/0b3aa14d1d71546c988a28e1b412981c1b80c8a1072e977a2f30c595cc4a/ruff-0.8.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:f05cdf8d050b30e2ba55c9b09330b51f9f97d36d4673213679b965d25a785f3c", size = 9909400 }, + { url = "https://files.pythonhosted.org/packages/23/08/f9f08cefb7921784c891c4151cce6ed357ff49e84b84978440cffbc87408/ruff-0.8.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:60f578c11feb1d3d257b2fb043ddb47501ab4816e7e221fbb0077f0d5d4e7b6f", size = 10766782 }, + { url = "https://files.pythonhosted.org/packages/e4/71/bf50c321ec179aa420c8ec40adac5ae9cc408d4d37283a485b19a2331ceb/ruff-0.8.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cbd5cf9b0ae8f30eebc7b360171bd50f59ab29d39f06a670b3e4501a36ba5897", size = 10286316 }, + { url = "https://files.pythonhosted.org/packages/f2/83/c82688a2a6117539aea0ce63fdf6c08e60fe0202779361223bcd7f40bd74/ruff-0.8.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b402ddee3d777683de60ff76da801fa7e5e8a71038f57ee53e903afbcefdaa58", size = 11338270 }, + { url = "https://files.pythonhosted.org/packages/7f/d7/bc6a45e5a22e627640388e703160afb1d77c572b1d0fda8b4349f334fc66/ruff-0.8.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:705832cd7d85605cb7858d8a13d75993c8f3ef1397b0831289109e953d833d29", size = 12058579 }, + { url = "https://files.pythonhosted.org/packages/da/3b/64150c93946ec851e6f1707ff586bb460ca671581380c919698d6a9267dc/ruff-0.8.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:32096b41aaf7a5cc095fa45b4167b890e4c8d3fd217603f3634c92a541de7248", size = 11615172 }, + { url = "https://files.pythonhosted.org/packages/e4/9e/cf12b697ea83cfe92ec4509ae414dc4c9b38179cc681a497031f0d0d9a8e/ruff-0.8.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e769083da9439508833cfc7c23e351e1809e67f47c50248250ce1ac52c21fb93", size = 12882398 }, + { url = "https://files.pythonhosted.org/packages/a9/27/96d10863accf76a9c97baceac30b0a52d917eb985a8ac058bd4636aeede0/ruff-0.8.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fe716592ae8a376c2673fdfc1f5c0c193a6d0411f90a496863c99cd9e2ae25d", size = 11176094 }, + { url = "https://files.pythonhosted.org/packages/eb/10/cd2fd77d4a4e7f03c29351be0f53278a393186b540b99df68beb5304fddd/ruff-0.8.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:81c148825277e737493242b44c5388a300584d73d5774defa9245aaef55448b0", size = 10771884 }, + { url = "https://files.pythonhosted.org/packages/71/5d/beabb2ff18870fc4add05fa3a69a4cb1b1d2d6f83f3cf3ae5ab0d52f455d/ruff-0.8.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d261d7850c8367704874847d95febc698a950bf061c9475d4a8b7689adc4f7fa", size = 10382535 }, + { url = "https://files.pythonhosted.org/packages/ae/29/6b3fdf3ad3e35b28d87c25a9ff4c8222ad72485ab783936b2b267250d7a7/ruff-0.8.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1ca4e3a87496dc07d2427b7dd7ffa88a1e597c28dad65ae6433ecb9f2e4f022f", size = 10886995 }, + { url = "https://files.pythonhosted.org/packages/e9/dc/859d889b4d9356a1a2cdbc1e4a0dda94052bc5b5300098647e51a58c430b/ruff-0.8.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:729850feed82ef2440aa27946ab39c18cb4a8889c1128a6d589ffa028ddcfc22", size = 11220750 }, + { url = "https://files.pythonhosted.org/packages/0b/08/e8f519f61f1d624264bfd6b8829e4c5f31c3c61193bc3cff1f19dbe7626a/ruff-0.8.2-py3-none-win32.whl", hash = "sha256:ac42caaa0411d6a7d9594363294416e0e48fc1279e1b0e948391695db2b3d5b1", size = 8729396 }, + { url = "https://files.pythonhosted.org/packages/f8/d4/ba1c7ab72aba37a2b71fe48ab95b80546dbad7a7f35ea28cf66fc5cea5f6/ruff-0.8.2-py3-none-win_amd64.whl", hash = "sha256:2aae99ec70abf43372612a838d97bfe77d45146254568d94926e8ed5bbb409ea", size = 9594729 }, + { url = "https://files.pythonhosted.org/packages/23/34/db20e12d3db11b8a2a8874258f0f6d96a9a4d631659d54575840557164c8/ruff-0.8.2-py3-none-win_arm64.whl", hash = "sha256:fb88e2a506b70cfbc2de6fae6681c4f944f7dd5f2fe87233a7233d888bad73e8", size = 9035131 }, ] [[package]] @@ -1518,6 +1949,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bf/65/813fc133609ebcb1299be6a42e5aea99d6344afb35ccb43f67e7daaa3b92/structlog-24.4.0-py3-none-any.whl", hash = "sha256:597f61e80a91cc0749a9fd2a098ed76715a1c8a01f73e336b746504d1aad7610", size = 67180 }, ] +[[package]] +name = "texttable" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/dc/0aff23d6036a4d3bf4f1d8c8204c5c79c4437e25e0ae94ffe4bbb55ee3c2/texttable-1.7.0.tar.gz", hash = "sha256:2d2068fb55115807d3ac77a4ca68fa48803e84ebb0ee2340f858107a36522638", size = 12831 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl", hash = "sha256:72227d592c82b3d7f672731ae73e4d1f88cd8e2ef5b075a7a7f01a23a3743917", size = 10768 }, +] + [[package]] name = "tomli" version = "2.0.1" diff --git a/lib/datautils/owid/datautils/decorators.py b/lib/datautils/owid/datautils/decorators.py index da5924a15da..f4046dd5f81 100644 --- a/lib/datautils/owid/datautils/decorators.py +++ b/lib/datautils/owid/datautils/decorators.py @@ -1,6 +1,5 @@ """Library decorators.""" - import functools import tempfile from typing import Any, Callable, Optional diff --git a/lib/datautils/owid/datautils/format/__init__.py b/lib/datautils/owid/datautils/format/__init__.py index 6144721b3de..2b5a43862e3 100644 --- a/lib/datautils/owid/datautils/format/__init__.py +++ b/lib/datautils/owid/datautils/format/__init__.py @@ -1,4 +1,5 @@ """Utils for the processing of different data formats.""" + from owid.datautils.format.numbers import format_number __all__ = [ diff --git a/lib/datautils/owid/datautils/format/numbers.py b/lib/datautils/owid/datautils/format/numbers.py index 3fd1d40d8ec..95b796263c5 100644 --- a/lib/datautils/owid/datautils/format/numbers.py +++ b/lib/datautils/owid/datautils/format/numbers.py @@ -1,4 +1,5 @@ """Numeric formatting.""" + import re from typing import Any, Dict, Set, Union diff --git a/lib/datautils/owid/datautils/google/__init__.py b/lib/datautils/owid/datautils/google/__init__.py index 10c831281f2..132a37db612 100644 --- a/lib/datautils/owid/datautils/google/__init__.py +++ b/lib/datautils/owid/datautils/google/__init__.py @@ -1,4 +1,5 @@ """Google utils.""" + from owid.datautils.google.api import GoogleApi __all__ = [ diff --git a/lib/datautils/owid/datautils/google/api.py b/lib/datautils/owid/datautils/google/api.py index 981b5e65a2a..4dd29996afc 100644 --- a/lib/datautils/owid/datautils/google/api.py +++ b/lib/datautils/owid/datautils/google/api.py @@ -1,4 +1,5 @@ """Google API class.""" + from typing import Any, Optional import gdown diff --git a/lib/datautils/owid/datautils/google/config.py b/lib/datautils/owid/datautils/google/config.py index 72c02cfb1e3..cb0ad9accb5 100644 --- a/lib/datautils/owid/datautils/google/config.py +++ b/lib/datautils/owid/datautils/google/config.py @@ -1,4 +1,5 @@ """Google configuration functions.""" + import os from pathlib import Path from shutil import copyfile diff --git a/lib/datautils/owid/datautils/google/sheets.py b/lib/datautils/owid/datautils/google/sheets.py index b966d2a1676..1050016ae53 100644 --- a/lib/datautils/owid/datautils/google/sheets.py +++ b/lib/datautils/owid/datautils/google/sheets.py @@ -1,4 +1,5 @@ """Google Sheet utils.""" + import os from typing import Any, Optional, Union diff --git a/lib/datautils/owid/datautils/io/__init__.py b/lib/datautils/owid/datautils/io/__init__.py index 60776d35c21..f50921da739 100644 --- a/lib/datautils/owid/datautils/io/__init__.py +++ b/lib/datautils/owid/datautils/io/__init__.py @@ -1,4 +1,5 @@ """Input/Output methods.""" + from owid.datautils.io.archive import decompress_file from owid.datautils.io.df import from_file as df_from_file from owid.datautils.io.df import to_file as df_to_file diff --git a/lib/datautils/owid/datautils/io/archive.py b/lib/datautils/owid/datautils/io/archive.py index e9c51495b6e..f0a4e1ec200 100644 --- a/lib/datautils/owid/datautils/io/archive.py +++ b/lib/datautils/owid/datautils/io/archive.py @@ -3,7 +3,9 @@ import tarfile import zipfile from pathlib import Path -from typing import Union +from typing import Union, cast + +from py7zr import SevenZipFile from owid.datautils.decorators import enable_file_download @@ -28,10 +30,17 @@ def decompress_file( Overwrite decompressed content if it already exists (otherwise raises an error if content already exists). """ + if isinstance(input_file, str): + input_file = Path(input_file) + input_file = cast(Path, input_file) + if zipfile.is_zipfile(input_file): _decompress_zip_file(input_file, output_folder, overwrite) elif tarfile.is_tarfile(input_file): _decompress_tar_file(input_file, output_folder, overwrite) + elif input_file.suffix.lower() == ".7z": + with SevenZipFile(input_file, mode="r") as z: + z.extractall(path=output_folder) else: raise ValueError("File is neither a zip nor a tar file.") diff --git a/lib/datautils/owid/datautils/io/df.py b/lib/datautils/owid/datautils/io/df.py index ffad5923a18..3b3251ee59e 100644 --- a/lib/datautils/owid/datautils/io/df.py +++ b/lib/datautils/owid/datautils/io/df.py @@ -1,4 +1,5 @@ """DataFrame io operations.""" + import inspect from pathlib import Path from typing import Any, List, Optional, Union diff --git a/lib/datautils/pyproject.toml b/lib/datautils/pyproject.toml index a173445e218..111eb2639ef 100644 --- a/lib/datautils/pyproject.toml +++ b/lib/datautils/pyproject.toml @@ -23,7 +23,8 @@ dependencies = [ "pydrive2>=1.15.0", "pyarrow>=10.0.1", "urllib3<2", - "click>=8.1.7" + "click>=8.1.7", + "py7zr>=0.22.0", ] [tool.uv] @@ -40,7 +41,7 @@ dev-dependencies = [ "openpyxl>=3.0.10", "lxml>=4.9.1", "pyright==1.1.373", - "ruff==0.1.6" + "ruff==0.8.2" ] [tool.ruff] diff --git a/lib/datautils/tests/google/test_config.py b/lib/datautils/tests/google/test_config.py index b1404869ea0..20a53d99964 100644 --- a/lib/datautils/tests/google/test_config.py +++ b/lib/datautils/tests/google/test_config.py @@ -29,16 +29,21 @@ def test_google_config_is_init_true(mock_check): def test_check_google_config_1(): """All files are created""" with tempfile.TemporaryDirectory() as config_dir: - with mock.patch("owid.datautils.google.config.CONFIG_DIR", config_dir), mock.patch( - "owid.datautils.google.config.CLIENT_SECRETS_PATH", - Path(config_dir) / "google_client_secrets.json", - ) as secrets_dir, mock.patch( - "owid.datautils.google.config.SETTINGS_PATH", - Path(config_dir) / "google_settings.yaml", - ) as settings_dir, mock.patch( - "owid.datautils.google.config.CREDENTIALS_PATH", - Path(config_dir) / "google_credentials.json", - ) as creds_dir: + with ( + mock.patch("owid.datautils.google.config.CONFIG_DIR", config_dir), + mock.patch( + "owid.datautils.google.config.CLIENT_SECRETS_PATH", + Path(config_dir) / "google_client_secrets.json", + ) as secrets_dir, + mock.patch( + "owid.datautils.google.config.SETTINGS_PATH", + Path(config_dir) / "google_settings.yaml", + ) as settings_dir, + mock.patch( + "owid.datautils.google.config.CREDENTIALS_PATH", + Path(config_dir) / "google_credentials.json", + ) as creds_dir, + ): with open(secrets_dir, "w") as f: f.write("This is test") with open(settings_dir, "w") as f: @@ -59,16 +64,21 @@ def test_check_google_config_2(): def test_check_google_config_3(): """Folder created, files not created""" with raises(FileNotFoundError), tempfile.TemporaryDirectory() as config_dir: - with mock.patch("owid.datautils.google.config.CONFIG_DIR", config_dir), mock.patch( - "owid.datautils.google.config.CLIENT_SECRETS_PATH", - Path(config_dir) / "google_client_secrets.json", - ) as _, mock.patch( - "owid.datautils.google.config.SETTINGS_PATH", - Path(config_dir) / "google_settings.yaml", - ) as _, mock.patch( - "owid.datautils.google.config.CREDENTIALS_PATH", - Path(config_dir) / "google_credentials.json", - ) as _: + with ( + mock.patch("owid.datautils.google.config.CONFIG_DIR", config_dir), + mock.patch( + "owid.datautils.google.config.CLIENT_SECRETS_PATH", + Path(config_dir) / "google_client_secrets.json", + ) as _, + mock.patch( + "owid.datautils.google.config.SETTINGS_PATH", + Path(config_dir) / "google_settings.yaml", + ) as _, + mock.patch( + "owid.datautils.google.config.CREDENTIALS_PATH", + Path(config_dir) / "google_credentials.json", + ) as _, + ): _check_google_config() @@ -85,15 +95,20 @@ def test_google_config_init_1(mocker_google_1, mocker_google_2): config_dir = next(tempfile._get_candidate_names()) # type: ignore defult_tmp_dir = tempfile._get_default_tempdir() # type: ignore config_dir = os.path.join(defult_tmp_dir, config_dir) - with mock.patch("owid.datautils.google.config.CONFIG_DIR", config_dir), mock.patch( - "owid.datautils.google.config.CLIENT_SECRETS_PATH", - Path(config_dir) / "google_client_secrets.json", - ) as _, mock.patch( - "owid.datautils.google.config.SETTINGS_PATH", - Path(config_dir) / "google_settings.yaml", - ) as _, mock.patch( - "owid.datautils.google.config.CREDENTIALS_PATH", - Path(config_dir) / "google_credentials.json", - ) as _: + with ( + mock.patch("owid.datautils.google.config.CONFIG_DIR", config_dir), + mock.patch( + "owid.datautils.google.config.CLIENT_SECRETS_PATH", + Path(config_dir) / "google_client_secrets.json", + ) as _, + mock.patch( + "owid.datautils.google.config.SETTINGS_PATH", + Path(config_dir) / "google_settings.yaml", + ) as _, + mock.patch( + "owid.datautils.google.config.CREDENTIALS_PATH", + Path(config_dir) / "google_credentials.json", + ) as _, + ): with tempfile.NamedTemporaryFile() as secrets_dir_og: google_config_init(str(secrets_dir_og.name)) diff --git a/lib/datautils/tests/io/test_df.py b/lib/datautils/tests/io/test_df.py index 7e8f29bf280..6c3929dcb97 100644 --- a/lib/datautils/tests/io/test_df.py +++ b/lib/datautils/tests/io/test_df.py @@ -1,6 +1,5 @@ -"""Test functions in owid.datautils.io.local module. +"""Test functions in owid.datautils.io.local module.""" -""" import tempfile from pathlib import Path from typing import Any diff --git a/lib/datautils/tests/io/test_json.py b/lib/datautils/tests/io/test_json.py index cca6a3f4a5b..c78540b40fc 100644 --- a/lib/datautils/tests/io/test_json.py +++ b/lib/datautils/tests/io/test_json.py @@ -1,6 +1,4 @@ -"""Test functions in owid.datautils.io.local module. - -""" +"""Test functions in owid.datautils.io.local module.""" from unittest.mock import mock_open, patch diff --git a/lib/datautils/tests/test_dataframes.py b/lib/datautils/tests/test_dataframes.py index d6b81a51c7b..a9cdb3eb9d8 100644 --- a/lib/datautils/tests/test_dataframes.py +++ b/lib/datautils/tests/test_dataframes.py @@ -1,6 +1,4 @@ -"""Test functions in owid.datautils.dataframes module. - -""" +"""Test functions in owid.datautils.dataframes module.""" import tempfile from pathlib import Path diff --git a/lib/datautils/tests/test_web.py b/lib/datautils/tests/test_web.py index b274fd32cc6..cd66c55caae 100644 --- a/lib/datautils/tests/test_web.py +++ b/lib/datautils/tests/test_web.py @@ -1,7 +1,4 @@ -"""Test functions in owid.datautils.web module. - -""" - +"""Test functions in owid.datautils.web module.""" from unittest import mock diff --git a/lib/datautils/uv.lock b/lib/datautils/uv.lock index 944630885df..fc38bb8d16b 100644 --- a/lib/datautils/uv.lock +++ b/lib/datautils/uv.lock @@ -117,6 +117,119 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/29/1d/9776890c7c4f070edb8d35ee986d9cde951e1f4187439f73f47922a6c6e9/botocore_stubs-1.35.20-py3-none-any.whl", hash = "sha256:f34d8908fa6a29d3dec57a373160d2defca3e2250d689864b23c9fea95669f97", size = 60132 }, ] +[[package]] +name = "brotli" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/c2/f9e977608bdf958650638c3f1e28f85a1b075f075ebbe77db8555463787b/Brotli-1.1.0.tar.gz", hash = "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724", size = 7372270 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/3a/dbf4fb970c1019a57b5e492e1e0eae745d32e59ba4d6161ab5422b08eefe/Brotli-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e1140c64812cb9b06c922e77f1c26a75ec5e3f0fb2bf92cc8c58720dec276752", size = 873045 }, + { url = "https://files.pythonhosted.org/packages/dd/11/afc14026ea7f44bd6eb9316d800d439d092c8d508752055ce8d03086079a/Brotli-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c8fd5270e906eef71d4a8d19b7c6a43760c6abcfcc10c9101d14eb2357418de9", size = 446218 }, + { url = "https://files.pythonhosted.org/packages/36/83/7545a6e7729db43cb36c4287ae388d6885c85a86dd251768a47015dfde32/Brotli-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ae56aca0402a0f9a3431cddda62ad71666ca9d4dc3a10a142b9dce2e3c0cda3", size = 2903872 }, + { url = "https://files.pythonhosted.org/packages/32/23/35331c4d9391fcc0f29fd9bec2c76e4b4eeab769afbc4b11dd2e1098fb13/Brotli-1.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:43ce1b9935bfa1ede40028054d7f48b5469cd02733a365eec8a329ffd342915d", size = 2941254 }, + { url = "https://files.pythonhosted.org/packages/3b/24/1671acb450c902edb64bd765d73603797c6c7280a9ada85a195f6b78c6e5/Brotli-1.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:7c4855522edb2e6ae7fdb58e07c3ba9111e7621a8956f481c68d5d979c93032e", size = 2857293 }, + { url = "https://files.pythonhosted.org/packages/d5/00/40f760cc27007912b327fe15bf6bfd8eaecbe451687f72a8abc587d503b3/Brotli-1.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:38025d9f30cf4634f8309c6874ef871b841eb3c347e90b0851f63d1ded5212da", size = 3002385 }, + { url = "https://files.pythonhosted.org/packages/b8/cb/8aaa83f7a4caa131757668c0fb0c4b6384b09ffa77f2fba9570d87ab587d/Brotli-1.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e6a904cb26bfefc2f0a6f240bdf5233be78cd2488900a2f846f3c3ac8489ab80", size = 2911104 }, + { url = "https://files.pythonhosted.org/packages/bc/c4/65456561d89d3c49f46b7fbeb8fe6e449f13bdc8ea7791832c5d476b2faf/Brotli-1.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a37b8f0391212d29b3a91a799c8e4a2855e0576911cdfb2515487e30e322253d", size = 2809981 }, + { url = "https://files.pythonhosted.org/packages/05/1b/cf49528437bae28abce5f6e059f0d0be6fecdcc1d3e33e7c54b3ca498425/Brotli-1.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e84799f09591700a4154154cab9787452925578841a94321d5ee8fb9a9a328f0", size = 2935297 }, + { url = "https://files.pythonhosted.org/packages/81/ff/190d4af610680bf0c5a09eb5d1eac6e99c7c8e216440f9c7cfd42b7adab5/Brotli-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f66b5337fa213f1da0d9000bc8dc0cb5b896b726eefd9c6046f699b169c41b9e", size = 2930735 }, + { url = "https://files.pythonhosted.org/packages/80/7d/f1abbc0c98f6e09abd3cad63ec34af17abc4c44f308a7a539010f79aae7a/Brotli-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5dab0844f2cf82be357a0eb11a9087f70c5430b2c241493fc122bb6f2bb0917c", size = 2933107 }, + { url = "https://files.pythonhosted.org/packages/34/ce/5a5020ba48f2b5a4ad1c0522d095ad5847a0be508e7d7569c8630ce25062/Brotli-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e4fe605b917c70283db7dfe5ada75e04561479075761a0b3866c081d035b01c1", size = 2845400 }, + { url = "https://files.pythonhosted.org/packages/44/89/fa2c4355ab1eecf3994e5a0a7f5492c6ff81dfcb5f9ba7859bd534bb5c1a/Brotli-1.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1e9a65b5736232e7a7f91ff3d02277f11d339bf34099a56cdab6a8b3410a02b2", size = 3031985 }, + { url = "https://files.pythonhosted.org/packages/af/a4/79196b4a1674143d19dca400866b1a4d1a089040df7b93b88ebae81f3447/Brotli-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:58d4b711689366d4a03ac7957ab8c28890415e267f9b6589969e74b6e42225ec", size = 2927099 }, + { url = "https://files.pythonhosted.org/packages/e9/54/1c0278556a097f9651e657b873ab08f01b9a9ae4cac128ceb66427d7cd20/Brotli-1.1.0-cp310-cp310-win32.whl", hash = "sha256:be36e3d172dc816333f33520154d708a2657ea63762ec16b62ece02ab5e4daf2", size = 333172 }, + { url = "https://files.pythonhosted.org/packages/f7/65/b785722e941193fd8b571afd9edbec2a9b838ddec4375d8af33a50b8dab9/Brotli-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:0c6244521dda65ea562d5a69b9a26120769b7a9fb3db2fe9545935ed6735b128", size = 357255 }, + { url = "https://files.pythonhosted.org/packages/96/12/ad41e7fadd5db55459c4c401842b47f7fee51068f86dd2894dd0dcfc2d2a/Brotli-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a3daabb76a78f829cafc365531c972016e4aa8d5b4bf60660ad8ecee19df7ccc", size = 873068 }, + { url = "https://files.pythonhosted.org/packages/95/4e/5afab7b2b4b61a84e9c75b17814198ce515343a44e2ed4488fac314cd0a9/Brotli-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c8146669223164fc87a7e3de9f81e9423c67a79d6b3447994dfb9c95da16e2d6", size = 446244 }, + { url = "https://files.pythonhosted.org/packages/9d/e6/f305eb61fb9a8580c525478a4a34c5ae1a9bcb12c3aee619114940bc513d/Brotli-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30924eb4c57903d5a7526b08ef4a584acc22ab1ffa085faceb521521d2de32dd", size = 2906500 }, + { url = "https://files.pythonhosted.org/packages/3e/4f/af6846cfbc1550a3024e5d3775ede1e00474c40882c7bf5b37a43ca35e91/Brotli-1.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ceb64bbc6eac5a140ca649003756940f8d6a7c444a68af170b3187623b43bebf", size = 2943950 }, + { url = "https://files.pythonhosted.org/packages/b3/e7/ca2993c7682d8629b62630ebf0d1f3bb3d579e667ce8e7ca03a0a0576a2d/Brotli-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a469274ad18dc0e4d316eefa616d1d0c2ff9da369af19fa6f3daa4f09671fd61", size = 2918527 }, + { url = "https://files.pythonhosted.org/packages/b3/96/da98e7bedc4c51104d29cc61e5f449a502dd3dbc211944546a4cc65500d3/Brotli-1.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:524f35912131cc2cabb00edfd8d573b07f2d9f21fa824bd3fb19725a9cf06327", size = 2845489 }, + { url = "https://files.pythonhosted.org/packages/e8/ef/ccbc16947d6ce943a7f57e1a40596c75859eeb6d279c6994eddd69615265/Brotli-1.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5b3cc074004d968722f51e550b41a27be656ec48f8afaeeb45ebf65b561481dd", size = 2914080 }, + { url = "https://files.pythonhosted.org/packages/80/d6/0bd38d758d1afa62a5524172f0b18626bb2392d717ff94806f741fcd5ee9/Brotli-1.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:19c116e796420b0cee3da1ccec3b764ed2952ccfcc298b55a10e5610ad7885f9", size = 2813051 }, + { url = "https://files.pythonhosted.org/packages/14/56/48859dd5d129d7519e001f06dcfbb6e2cf6db92b2702c0c2ce7d97e086c1/Brotli-1.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:510b5b1bfbe20e1a7b3baf5fed9e9451873559a976c1a78eebaa3b86c57b4265", size = 2938172 }, + { url = "https://files.pythonhosted.org/packages/3d/77/a236d5f8cd9e9f4348da5acc75ab032ab1ab2c03cc8f430d24eea2672888/Brotli-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a1fd8a29719ccce974d523580987b7f8229aeace506952fa9ce1d53a033873c8", size = 2933023 }, + { url = "https://files.pythonhosted.org/packages/f1/87/3b283efc0f5cb35f7f84c0c240b1e1a1003a5e47141a4881bf87c86d0ce2/Brotli-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c247dd99d39e0338a604f8c2b3bc7061d5c2e9e2ac7ba9cc1be5a69cb6cd832f", size = 2935871 }, + { url = "https://files.pythonhosted.org/packages/f3/eb/2be4cc3e2141dc1a43ad4ca1875a72088229de38c68e842746b342667b2a/Brotli-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1b2c248cd517c222d89e74669a4adfa5577e06ab68771a529060cf5a156e9757", size = 2847784 }, + { url = "https://files.pythonhosted.org/packages/66/13/b58ddebfd35edde572ccefe6890cf7c493f0c319aad2a5badee134b4d8ec/Brotli-1.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:2a24c50840d89ded6c9a8fdc7b6ed3692ed4e86f1c4a4a938e1e92def92933e0", size = 3034905 }, + { url = "https://files.pythonhosted.org/packages/84/9c/bc96b6c7db824998a49ed3b38e441a2cae9234da6fa11f6ed17e8cf4f147/Brotli-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f31859074d57b4639318523d6ffdca586ace54271a73ad23ad021acd807eb14b", size = 2929467 }, + { url = "https://files.pythonhosted.org/packages/e7/71/8f161dee223c7ff7fea9d44893fba953ce97cf2c3c33f78ba260a91bcff5/Brotli-1.1.0-cp311-cp311-win32.whl", hash = "sha256:39da8adedf6942d76dc3e46653e52df937a3c4d6d18fdc94a7c29d263b1f5b50", size = 333169 }, + { url = "https://files.pythonhosted.org/packages/02/8a/fece0ee1057643cb2a5bbf59682de13f1725f8482b2c057d4e799d7ade75/Brotli-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:aac0411d20e345dc0920bdec5548e438e999ff68d77564d5e9463a7ca9d3e7b1", size = 357253 }, + { url = "https://files.pythonhosted.org/packages/5c/d0/5373ae13b93fe00095a58efcbce837fd470ca39f703a235d2a999baadfbc/Brotli-1.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:32d95b80260d79926f5fab3c41701dbb818fde1c9da590e77e571eefd14abe28", size = 815693 }, + { url = "https://files.pythonhosted.org/packages/8e/48/f6e1cdf86751300c288c1459724bfa6917a80e30dbfc326f92cea5d3683a/Brotli-1.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b760c65308ff1e462f65d69c12e4ae085cff3b332d894637f6273a12a482d09f", size = 422489 }, + { url = "https://files.pythonhosted.org/packages/06/88/564958cedce636d0f1bed313381dfc4b4e3d3f6015a63dae6146e1b8c65c/Brotli-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409", size = 873081 }, + { url = "https://files.pythonhosted.org/packages/58/79/b7026a8bb65da9a6bb7d14329fd2bd48d2b7f86d7329d5cc8ddc6a90526f/Brotli-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2", size = 446244 }, + { url = "https://files.pythonhosted.org/packages/e5/18/c18c32ecea41b6c0004e15606e274006366fe19436b6adccc1ae7b2e50c2/Brotli-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451", size = 2906505 }, + { url = "https://files.pythonhosted.org/packages/08/c8/69ec0496b1ada7569b62d85893d928e865df29b90736558d6c98c2031208/Brotli-1.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7f4bf76817c14aa98cc6697ac02f3972cb8c3da93e9ef16b9c66573a68014f91", size = 2944152 }, + { url = "https://files.pythonhosted.org/packages/ab/fb/0517cea182219d6768113a38167ef6d4eb157a033178cc938033a552ed6d/Brotli-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0c5516f0aed654134a2fc936325cc2e642f8a0e096d075209672eb321cff408", size = 2919252 }, + { url = "https://files.pythonhosted.org/packages/c7/53/73a3431662e33ae61a5c80b1b9d2d18f58dfa910ae8dd696e57d39f1a2f5/Brotli-1.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c3020404e0b5eefd7c9485ccf8393cfb75ec38ce75586e046573c9dc29967a0", size = 2845955 }, + { url = "https://files.pythonhosted.org/packages/55/ac/bd280708d9c5ebdbf9de01459e625a3e3803cce0784f47d633562cf40e83/Brotli-1.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4ed11165dd45ce798d99a136808a794a748d5dc38511303239d4e2363c0695dc", size = 2914304 }, + { url = "https://files.pythonhosted.org/packages/76/58/5c391b41ecfc4527d2cc3350719b02e87cb424ef8ba2023fb662f9bf743c/Brotli-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180", size = 2814452 }, + { url = "https://files.pythonhosted.org/packages/c7/4e/91b8256dfe99c407f174924b65a01f5305e303f486cc7a2e8a5d43c8bec3/Brotli-1.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248", size = 2938751 }, + { url = "https://files.pythonhosted.org/packages/5a/a6/e2a39a5d3b412938362bbbeba5af904092bf3f95b867b4a3eb856104074e/Brotli-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966", size = 2933757 }, + { url = "https://files.pythonhosted.org/packages/13/f0/358354786280a509482e0e77c1a5459e439766597d280f28cb097642fc26/Brotli-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87a3044c3a35055527ac75e419dfa9f4f3667a1e887ee80360589eb8c90aabb9", size = 2936146 }, + { url = "https://files.pythonhosted.org/packages/80/f7/daf538c1060d3a88266b80ecc1d1c98b79553b3f117a485653f17070ea2a/Brotli-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c5529b34c1c9d937168297f2c1fde7ebe9ebdd5e121297ff9c043bdb2ae3d6fb", size = 2848055 }, + { url = "https://files.pythonhosted.org/packages/ad/cf/0eaa0585c4077d3c2d1edf322d8e97aabf317941d3a72d7b3ad8bce004b0/Brotli-1.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ca63e1890ede90b2e4454f9a65135a4d387a4585ff8282bb72964fab893f2111", size = 3035102 }, + { url = "https://files.pythonhosted.org/packages/d8/63/1c1585b2aa554fe6dbce30f0c18bdbc877fa9a1bf5ff17677d9cca0ac122/Brotli-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e79e6520141d792237c70bcd7a3b122d00f2613769ae0cb61c52e89fd3443839", size = 2930029 }, + { url = "https://files.pythonhosted.org/packages/5f/3b/4e3fd1893eb3bbfef8e5a80d4508bec17a57bb92d586c85c12d28666bb13/Brotli-1.1.0-cp312-cp312-win32.whl", hash = "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0", size = 333276 }, + { url = "https://files.pythonhosted.org/packages/3d/d5/942051b45a9e883b5b6e98c041698b1eb2012d25e5948c58d6bf85b1bb43/Brotli-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951", size = 357255 }, + { url = "https://files.pythonhosted.org/packages/0a/9f/fb37bb8ffc52a8da37b1c03c459a8cd55df7a57bdccd8831d500e994a0ca/Brotli-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8bf32b98b75c13ec7cf774164172683d6e7891088f6316e54425fde1efc276d5", size = 815681 }, + { url = "https://files.pythonhosted.org/packages/06/b3/dbd332a988586fefb0aa49c779f59f47cae76855c2d00f450364bb574cac/Brotli-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bc37c4d6b87fb1017ea28c9508b36bbcb0c3d18b4260fcdf08b200c74a6aee8", size = 422475 }, + { url = "https://files.pythonhosted.org/packages/bb/80/6aaddc2f63dbcf2d93c2d204e49c11a9ec93a8c7c63261e2b4bd35198283/Brotli-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c0ef38c7a7014ffac184db9e04debe495d317cc9c6fb10071f7fefd93100a4f", size = 2906173 }, + { url = "https://files.pythonhosted.org/packages/ea/1d/e6ca79c96ff5b641df6097d299347507d39a9604bde8915e76bf026d6c77/Brotli-1.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91d7cc2a76b5567591d12c01f019dd7afce6ba8cba6571187e21e2fc418ae648", size = 2943803 }, + { url = "https://files.pythonhosted.org/packages/ac/a3/d98d2472e0130b7dd3acdbb7f390d478123dbf62b7d32bda5c830a96116d/Brotli-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a93dde851926f4f2678e704fadeb39e16c35d8baebd5252c9fd94ce8ce68c4a0", size = 2918946 }, + { url = "https://files.pythonhosted.org/packages/c4/a5/c69e6d272aee3e1423ed005d8915a7eaa0384c7de503da987f2d224d0721/Brotli-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0db75f47be8b8abc8d9e31bc7aad0547ca26f24a54e6fd10231d623f183d089", size = 2845707 }, + { url = "https://files.pythonhosted.org/packages/58/9f/4149d38b52725afa39067350696c09526de0125ebfbaab5acc5af28b42ea/Brotli-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6967ced6730aed543b8673008b5a391c3b1076d834ca438bbd70635c73775368", size = 2936231 }, + { url = "https://files.pythonhosted.org/packages/5a/5a/145de884285611838a16bebfdb060c231c52b8f84dfbe52b852a15780386/Brotli-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7eedaa5d036d9336c95915035fb57422054014ebdeb6f3b42eac809928e40d0c", size = 2848157 }, + { url = "https://files.pythonhosted.org/packages/50/ae/408b6bfb8525dadebd3b3dd5b19d631da4f7d46420321db44cd99dcf2f2c/Brotli-1.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d487f5432bf35b60ed625d7e1b448e2dc855422e87469e3f450aa5552b0eb284", size = 3035122 }, + { url = "https://files.pythonhosted.org/packages/af/85/a94e5cfaa0ca449d8f91c3d6f78313ebf919a0dbd55a100c711c6e9655bc/Brotli-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832436e59afb93e1836081a20f324cb185836c617659b07b129141a8426973c7", size = 2930206 }, + { url = "https://files.pythonhosted.org/packages/c2/f0/a61d9262cd01351df22e57ad7c34f66794709acab13f34be2675f45bf89d/Brotli-1.1.0-cp313-cp313-win32.whl", hash = "sha256:43395e90523f9c23a3d5bdf004733246fba087f2948f87ab28015f12359ca6a0", size = 333804 }, + { url = "https://files.pythonhosted.org/packages/7e/c1/ec214e9c94000d1c1974ec67ced1c970c148aa6b8d8373066123fc3dbf06/Brotli-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:9011560a466d2eb3f5a6e4929cf4a09be405c64154e12df0dd72713f6500e32b", size = 358517 }, + { url = "https://files.pythonhosted.org/packages/1b/aa/aa6e0c9848ee4375514af0b27abf470904992939b7363ae78fc8aca8a9a8/Brotli-1.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5fb2ce4b8045c78ebbc7b8f3c15062e435d47e7393cc57c25115cfd49883747a", size = 873048 }, + { url = "https://files.pythonhosted.org/packages/ae/32/38bba1a8bef9ecb1cda08439fd28d7e9c51aff13b4783a4f1610da90b6c2/Brotli-1.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7905193081db9bfa73b1219140b3d315831cbff0d8941f22da695832f0dd188f", size = 446207 }, + { url = "https://files.pythonhosted.org/packages/3c/6a/14cc20ddc53efc274601c8195791a27cfb7acc5e5134e0f8c493a8b8821a/Brotli-1.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a77def80806c421b4b0af06f45d65a136e7ac0bdca3c09d9e2ea4e515367c7e9", size = 2903803 }, + { url = "https://files.pythonhosted.org/packages/9a/26/62b2d894d4e82d7a7f4e0bb9007a42bbc765697a5679b43186acd68d7a79/Brotli-1.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8dadd1314583ec0bf2d1379f7008ad627cd6336625d6679cf2f8e67081b83acf", size = 2941149 }, + { url = "https://files.pythonhosted.org/packages/a9/ca/00d55bbdd8631236c61777742d8a8454cf6a87eb4125cad675912c68bec7/Brotli-1.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:901032ff242d479a0efa956d853d16875d42157f98951c0230f69e69f9c09bac", size = 2672253 }, + { url = "https://files.pythonhosted.org/packages/e2/e6/4a730f6e5b5d538e92d09bc51bf69119914f29a222f9e1d65ae4abb27a4e/Brotli-1.1.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:22fc2a8549ffe699bfba2256ab2ed0421a7b8fadff114a3d201794e45a9ff578", size = 2757005 }, + { url = "https://files.pythonhosted.org/packages/cb/6b/8cf297987fe3c1bf1c87f0c0b714af2ce47092b8d307b9f6ecbc65f98968/Brotli-1.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ae15b066e5ad21366600ebec29a7ccbc86812ed267e4b28e860b8ca16a2bc474", size = 2910658 }, + { url = "https://files.pythonhosted.org/packages/2c/1f/be9443995821c933aad7159803f84ef4923c6f5b72c2affd001192b310fc/Brotli-1.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:949f3b7c29912693cee0afcf09acd6ebc04c57af949d9bf77d6101ebb61e388c", size = 2809728 }, + { url = "https://files.pythonhosted.org/packages/76/2f/213bab6efa902658c80a1247142d42b138a27ccdd6bade49ca9cd74e714a/Brotli-1.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:89f4988c7203739d48c6f806f1e87a1d96e0806d44f0fba61dba81392c9e474d", size = 2935043 }, + { url = "https://files.pythonhosted.org/packages/27/89/bbb14fa98e895d1e601491fba54a5feec167d262f0d3d537a3b0d4cd0029/Brotli-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:de6551e370ef19f8de1807d0a9aa2cdfdce2e85ce88b122fe9f6b2b076837e59", size = 2930639 }, + { url = "https://files.pythonhosted.org/packages/14/87/03a6d6e1866eddf9f58cc57e35befbeb5514da87a416befe820150cae63f/Brotli-1.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0737ddb3068957cf1b054899b0883830bb1fec522ec76b1098f9b6e0f02d9419", size = 2932834 }, + { url = "https://files.pythonhosted.org/packages/a4/d5/e5f85e04f75144d1a89421ba432def6bdffc8f28b04f5b7d540bbd03362c/Brotli-1.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4f3607b129417e111e30637af1b56f24f7a49e64763253bbc275c75fa887d4b2", size = 2845213 }, + { url = "https://files.pythonhosted.org/packages/99/bf/25ef07add7afbb1aacd4460726a1a40370dfd60c0810b6f242a6d3871d7e/Brotli-1.1.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:6c6e0c425f22c1c719c42670d561ad682f7bfeeef918edea971a79ac5252437f", size = 3031573 }, + { url = "https://files.pythonhosted.org/packages/55/22/948a97bda5c9dc9968d56b9ed722d9727778db43739cf12ef26ff69be94d/Brotli-1.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:494994f807ba0b92092a163a0a283961369a65f6cbe01e8891132b7a320e61eb", size = 2926885 }, + { url = "https://files.pythonhosted.org/packages/31/ba/e53d107399b535ef89deb6977dd8eae468e2dde7b1b74c6cbe2c0e31fda2/Brotli-1.1.0-cp39-cp39-win32.whl", hash = "sha256:f0d8a7a6b5983c2496e364b969f0e526647a06b075d034f3297dc66f3b360c64", size = 333171 }, + { url = "https://files.pythonhosted.org/packages/99/b3/f7b3af539f74b82e1c64d28685a5200c631cc14ae751d37d6ed819655627/Brotli-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:cdad5b9014d83ca68c25d2e9444e28e967ef16e80f6b436918c700c117a85467", size = 357258 }, +] + +[[package]] +name = "brotlicffi" +version = "1.1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/9d/70caa61192f570fcf0352766331b735afa931b4c6bc9a348a0925cc13288/brotlicffi-1.1.0.0.tar.gz", hash = "sha256:b77827a689905143f87915310b93b273ab17888fd43ef350d4832c4a71083c13", size = 465192 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/11/7b96009d3dcc2c931e828ce1e157f03824a69fb728d06bfd7b2fc6f93718/brotlicffi-1.1.0.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9b7ae6bd1a3f0df532b6d67ff674099a96d22bc0948955cb338488c31bfb8851", size = 453786 }, + { url = "https://files.pythonhosted.org/packages/d6/e6/a8f46f4a4ee7856fbd6ac0c6fb0dc65ed181ba46cd77875b8d9bbe494d9e/brotlicffi-1.1.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19ffc919fa4fc6ace69286e0a23b3789b4219058313cf9b45625016bf7ff996b", size = 2911165 }, + { url = "https://files.pythonhosted.org/packages/be/20/201559dff14e83ba345a5ec03335607e47467b6633c210607e693aefac40/brotlicffi-1.1.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9feb210d932ffe7798ee62e6145d3a757eb6233aa9a4e7db78dd3690d7755814", size = 2927895 }, + { url = "https://files.pythonhosted.org/packages/cd/15/695b1409264143be3c933f708a3f81d53c4a1e1ebbc06f46331decbf6563/brotlicffi-1.1.0.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84763dbdef5dd5c24b75597a77e1b30c66604725707565188ba54bab4f114820", size = 2851834 }, + { url = "https://files.pythonhosted.org/packages/b4/40/b961a702463b6005baf952794c2e9e0099bde657d0d7e007f923883b907f/brotlicffi-1.1.0.0-cp37-abi3-win32.whl", hash = "sha256:1b12b50e07c3911e1efa3a8971543e7648100713d4e0971b13631cce22c587eb", size = 341731 }, + { url = "https://files.pythonhosted.org/packages/1c/fa/5408a03c041114ceab628ce21766a4ea882aa6f6f0a800e04ee3a30ec6b9/brotlicffi-1.1.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:994a4f0681bb6c6c3b0925530a1926b7a189d878e6e5e38fae8efa47c5d9c613", size = 366783 }, + { url = "https://files.pythonhosted.org/packages/e5/3b/bd4f3d2bcf2306ae66b0346f5b42af1962480b200096ffc7abc3bd130eca/brotlicffi-1.1.0.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:2e4aeb0bd2540cb91b069dbdd54d458da8c4334ceaf2d25df2f4af576d6766ca", size = 397397 }, + { url = "https://files.pythonhosted.org/packages/54/10/1fd57864449360852c535c2381ee7120ba8f390aa3869df967c44ca7eba1/brotlicffi-1.1.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b7b0033b0d37bb33009fb2fef73310e432e76f688af76c156b3594389d81391", size = 379698 }, + { url = "https://files.pythonhosted.org/packages/e5/95/15aa422aa6450e6556e54a5fd1650ff59f470aed77ac739aa90ab63dc611/brotlicffi-1.1.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54a07bb2374a1eba8ebb52b6fafffa2afd3c4df85ddd38fcc0511f2bb387c2a8", size = 378635 }, + { url = "https://files.pythonhosted.org/packages/6c/a7/f254e13b2cb43337d6d99a4ec10394c134e41bfda8a2eff15b75627f4a3d/brotlicffi-1.1.0.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7901a7dc4b88f1c1475de59ae9be59799db1007b7d059817948d8e4f12e24e35", size = 385719 }, + { url = "https://files.pythonhosted.org/packages/72/a9/0971251c4427c14b2a827dba3d910d4d3330dabf23d4278bf6d06a978847/brotlicffi-1.1.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ce01c7316aebc7fce59da734286148b1d1b9455f89cf2c8a4dfce7d41db55c2d", size = 361760 }, + { url = "https://files.pythonhosted.org/packages/35/9b/e0b577351e1d9d5890e1a56900c4ceaaef783b807145cd229446a43cf437/brotlicffi-1.1.0.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1a807d760763e398bbf2c6394ae9da5815901aa93ee0a37bca5efe78d4ee3171", size = 397392 }, + { url = "https://files.pythonhosted.org/packages/4f/7f/a16534d28386f74781db8b4544a764cf955abae336379a76f50e745bb0ee/brotlicffi-1.1.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fa8ca0623b26c94fccc3a1fdd895be1743b838f3917300506d04aa3346fd2a14", size = 379695 }, + { url = "https://files.pythonhosted.org/packages/50/2a/699388b5e489726991132441b55aff0691dd73c49105ef220408a5ab98d6/brotlicffi-1.1.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3de0cf28a53a3238b252aca9fed1593e9d36c1d116748013339f0949bfc84112", size = 378629 }, + { url = "https://files.pythonhosted.org/packages/4a/3f/58254e7fbe6011bf043e4dcade0e16995a9f82b731734fad97220d201f42/brotlicffi-1.1.0.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6be5ec0e88a4925c91f3dea2bb0013b3a2accda6f77238f76a34a1ea532a1cb0", size = 385712 }, + { url = "https://files.pythonhosted.org/packages/40/16/2a29a625a6f74d13726387f83484dfaaf6fcdaafaadfbe26a0412ae268cc/brotlicffi-1.1.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d9eb71bb1085d996244439154387266fd23d6ad37161f6f52f1cd41dd95a3808", size = 361747 }, +] + [[package]] name = "bump2version" version = "1.0.1" @@ -673,6 +786,54 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/d9/a1e041c5e7caa9a05c925f4bdbdfb7f006d1f74996af53467bc394c97be7/importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b", size = 26514 }, ] +[[package]] +name = "inflate64" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/99/18f9940d4a3f2cabc4396a587ddf1bd93236bdb372d9e78e2b0365e40990/inflate64-1.0.0.tar.gz", hash = "sha256:3278827b803cf006a1df251f3e13374c7d26db779e5a33329cc11789b804bc2d", size = 895853 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/cf/06af80e81dd4bbb7e883291cf1726035d526f066a37c4ed4d4cd88a7a49d/inflate64-1.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a90c0bdf4a7ecddd8a64cc977181810036e35807f56b0bcacee9abb0fcfd18dc", size = 59418 }, + { url = "https://files.pythonhosted.org/packages/c9/4b/6f18918220b1a8e935121cece1dc917e62fa593fc637a621470f9b9a601a/inflate64-1.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:57fe7c14aebf1c5a74fc3b70d355be1280a011521a76aa3895486e62454f4242", size = 36231 }, + { url = "https://files.pythonhosted.org/packages/aa/f4/f4b5dbd78dd5af66b6ca32778ebaa9c14d67b68ea84e96592ccf40786a41/inflate64-1.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d90730165f471d61a1a694a5e354f3ffa938227e8dcecb62d5d728e8069cee94", size = 35738 }, + { url = "https://files.pythonhosted.org/packages/10/23/26289a700550767cf5eb7550f78ad826529706287393f224bbaee3c1b1e2/inflate64-1.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:543f400201f5c101141af3c79c82059e1aa6ef4f1584a7f1fa035fb2e465097f", size = 92855 }, + { url = "https://files.pythonhosted.org/packages/b8/f4/e387a50f5027194eac4f9712d57b97e3e1a012402eaae98bcf1ebe8a97d1/inflate64-1.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ceca14f7ec19fb44b047f56c50efb7521b389d222bba2b0a10286a0caeb03fa", size = 93141 }, + { url = "https://files.pythonhosted.org/packages/33/c8/e516aecd9ed0dc75d8df041ed4ef80f2e2be39d0e516c7269b7f274e760a/inflate64-1.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b559937a42f0c175b4d2dfc7eb53b97bdc87efa9add15ed5549c6abc1e89d02f", size = 95262 }, + { url = "https://files.pythonhosted.org/packages/0b/aa/ed3ab5f8c13afc432fb382edf97cede7a6f9be73ecf98bfe64b686c8d223/inflate64-1.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5ff8bd2a562343fcbc4eea26fdc368904a3b5f6bb8262344274d3d74a1de15bb", size = 95912 }, + { url = "https://files.pythonhosted.org/packages/e0/64/5637c4f67ed15518c0765b85b528ed79536caaf8ba167a9f7173e334d4a8/inflate64-1.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:0fe481f31695d35a433c3044ac8fd5d9f5069aaad03a0c04b570eb258ce655aa", size = 35166 }, + { url = "https://files.pythonhosted.org/packages/af/92/701b3c76b1cf244026c3e78dff8487955cf6960c1d9f350e2820a0d1a5d9/inflate64-1.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a45f6979ad5874d4d4898c2fc770b136e61b96b850118fdaec5a5af1b9123a", size = 59450 }, + { url = "https://files.pythonhosted.org/packages/bb/1d/af0253fafc27cadd29e3b111ebb3011b8c913a3554b403c90c7595f5933e/inflate64-1.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:022ca1cc928e7365a05f7371ff06af143c6c667144965e2cf9a9236a2ae1c291", size = 36267 }, + { url = "https://files.pythonhosted.org/packages/b6/22/7949030be11f4754bd6ed7067e9bebdf614013b89ccd4638330a85821b51/inflate64-1.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:46792ecf3565d64fd2c519b0a780c03a57e195613c9954ef94e739a057b3fd06", size = 35740 }, + { url = "https://files.pythonhosted.org/packages/e4/87/c6ce0093a345c04811f6171a367665dec17dcc4617ca150dd37e9ae7bd33/inflate64-1.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a70ea2e456c15f7aa7c74b8ab8f20b4f8940ec657604c9f0a9de3342f280fff", size = 95896 }, + { url = "https://files.pythonhosted.org/packages/62/d6/fe113b12773cad2c093d381c2b1629f9cfa240c9ad86a7f9f9079e7a51b5/inflate64-1.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e243ea9bd36a035059f2365bd6d156ff59717fbafb0255cb0c75bf151bf6904", size = 96007 }, + { url = "https://files.pythonhosted.org/packages/f0/a6/9165bee4b7fc5af949fec12a2cea7ad73bf9ee97dfb96a0276274c48e709/inflate64-1.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4dc392dec1cd11cacda3d2637214ca45e38202e8a4f31d4a4e566d6e90625fc4", size = 98297 }, + { url = "https://files.pythonhosted.org/packages/ee/72/0aeb360101eeed32696fc6c623bc1780fac895a9fc2e93b582cb1e22ca54/inflate64-1.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8b402a50eda7ee75f342fc346d33a41bca58edc222a4b17f9be0db1daed459fa", size = 98858 }, + { url = "https://files.pythonhosted.org/packages/94/4a/8301ad59b57d9de504b0fdce22bf980dfb231753e6d7aed12af938f7f9fd/inflate64-1.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:f5924499dc8800928c0ee4580fa8eb4ffa880b2cce4431537d0390e503a9c9ee", size = 35167 }, + { url = "https://files.pythonhosted.org/packages/18/82/47021b8919c1dc276d0502296f15ffac1cd648b94b35cadb14cb812b6199/inflate64-1.0.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0c644bf7208e20825ca3bbb5fb1f7f495cfcb49eb01a5f67338796d44a42f2bf", size = 59509 }, + { url = "https://files.pythonhosted.org/packages/e0/c9/00701be8e48dc9c9b9488001d9c66d6cb6f6bb0c48af9abf33a69726d130/inflate64-1.0.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9964a4eaf26a9d36f82a1d9b12c28e35800dd3d99eb340453ed12ac90c2976a8", size = 36305 }, + { url = "https://files.pythonhosted.org/packages/25/c0/11dea5e298b2e7d61f0fbd1005553e8796e35536751980b676547fcc57ef/inflate64-1.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2cccded63865640d03253897be7232b2bbac295fe43914c61f86a57aa23bb61d", size = 35756 }, + { url = "https://files.pythonhosted.org/packages/86/ba/4debdaaafdc21853621caf463a498a754ee4352893454c596dbd65294e9f/inflate64-1.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d491f104fb3701926ebd82b8c9250dfba0ddcab584504e26f1e4adb26730378d", size = 96127 }, + { url = "https://files.pythonhosted.org/packages/89/81/8f559c199ec13d0b70d0dc46811490b2976873c96c564941583777e9b343/inflate64-1.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ebad4a6cd2a2c1d81be0b09d4006479f3b258803c49a9224ef8ca0b649072fa", size = 96903 }, + { url = "https://files.pythonhosted.org/packages/46/41/39ac4c7e17d0690578b716a0ff34e00600616994795b0645fd61fc600c0f/inflate64-1.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:6823b2c0cff3a8159140f3b17ec64fb8ec0e663b45a6593618ecdde8aeecb5b2", size = 98855 }, + { url = "https://files.pythonhosted.org/packages/44/dd/be5d69492c180f94a6af8a15564ce365bdcb84bd1a6fb32949d6913959aa/inflate64-1.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:228d504239d27958e71fc77e3119a6ac4528127df38468a0c95a5bd3927204b8", size = 99884 }, + { url = "https://files.pythonhosted.org/packages/8c/0d/a5266bd4f2cdb7fad1eae3ffe4dcc16f9769323660a0a6cfbe9cc1d2cf03/inflate64-1.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:ae2572e06bcfe15e3bbf77d4e4a6d6c55e2a70d6abceaaf60c5c3653ddb96dfd", size = 35334 }, + { url = "https://files.pythonhosted.org/packages/04/1c/47f9c93df339c381f3f3e7b983d7abf0756f8bd227bf9fb5a2e8b09ea9e5/inflate64-1.0.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bf2981b95c1f26242bb084d9a07f3feb0cfe3d6d0a8d90f42389803bc1252c4a", size = 59410 }, + { url = "https://files.pythonhosted.org/packages/b9/28/bde1595cae0379c521ae74f698f7da1345d990d2df7558df59470ced639b/inflate64-1.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9373ccf0661cc72ac84a0ad622634144da5ce7d57c9572ed0723d67a149feed2", size = 36230 }, + { url = "https://files.pythonhosted.org/packages/13/64/4350547de7c8a3ac27c1a0ab5807ac2fcbcde121c44190f87da1039421eb/inflate64-1.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e4650c6f65011ec57cf5cd96b92d5b7c6f59e502930c86eb8227c93cf02dc270", size = 35739 }, + { url = "https://files.pythonhosted.org/packages/1a/d6/b55e43c3503d7780aabe3d1d5493c983a99947dff228d9b622d49c58af68/inflate64-1.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a475e8822f1a74c873e60b8f270773757ade024097ca39e43402d47c049c67d4", size = 92655 }, + { url = "https://files.pythonhosted.org/packages/c0/c0/417e5183543445818930b3fe181d718e519d26a227b5b77871d8f0c8502d/inflate64-1.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4367480733ac8daf368f6fc704b7c9db85521ee745eb5bd443f4b97d2051acc", size = 92966 }, + { url = "https://files.pythonhosted.org/packages/6d/8e/ada659c83abb78222c666bb8d35b4791cca25a8a4a750f4bc457402d2430/inflate64-1.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6c5775c91f94f5eced9160fb0af12a09f3e030194f91a6a46e706a79350bd056", size = 95075 }, + { url = "https://files.pythonhosted.org/packages/7f/15/59c0e45e091599a05a098374166ff74d3da5f9c0ffa72e8c78b7f0fcaa2b/inflate64-1.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d76d205b844d78ce04768060084ef20e64dcc63a3e9166674f857acaf4d140ed", size = 95751 }, + { url = "https://files.pythonhosted.org/packages/80/21/a90d085b9ea67729d5cef57566c69fd4dec8205c22415cde4be62b7adc64/inflate64-1.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:92f0dc6af0e8e97324981178dc442956cbff1247a56d1e201af8d865244653f8", size = 35164 }, + { url = "https://files.pythonhosted.org/packages/53/91/43238dd8a7e5bab71abae872c09931db4b31aebf672afccb305f79aacb3e/inflate64-1.0.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f79542478e49e471e8b23556700e6f688a40dc93e9a746f77a546c13251b59b1", size = 34648 }, + { url = "https://files.pythonhosted.org/packages/ef/6f/ce090934a80c1fd0b5b07c125ed6eb2845f11a78af344d69c0f051dcab97/inflate64-1.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a270be6b10cde01258c0097a663a307c62d12c78eb8f62f8e29f205335942c9", size = 36473 }, + { url = "https://files.pythonhosted.org/packages/b4/fe/2cd4bf78696213b807860002c182dd1751ba52c1559143b1b8daa7904733/inflate64-1.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1616a87ff04f583e9558cc247ec0b72a30d540ee0c17cc77823be175c0ec92f0", size = 36478 }, + { url = "https://files.pythonhosted.org/packages/43/dd/e62444c0ef7d1228b622e6d3dacf9ea237d8807a78619a83832a3b4a5adf/inflate64-1.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:137ca6b315f0157a786c3a755a09395ca69aed8bcf42ad3437cb349f5ebc86d2", size = 35630 }, + { url = "https://files.pythonhosted.org/packages/2b/1e/0e346cb5eced6f2f30bb110e6d7c7ee7ab70ad1ff44b743d4cf314f60f7d/inflate64-1.0.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:dd6d3e7d47df43210a995fd1f5989602b64de3f2a17cf4cbff553518b3577fd4", size = 34644 }, + { url = "https://files.pythonhosted.org/packages/d4/f7/7ac502391ea56af23661a707e9c94efc01376a27c1b06f7a0cfb553c0f17/inflate64-1.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f033b2879696b855200cde5ca4e293132c7499df790acb2c0dacb336d5e83b1", size = 36467 }, + { url = "https://files.pythonhosted.org/packages/09/ea/7c3e7c856f42bee178a9fea816e259b02bd4198873cf07b5a839b96d9696/inflate64-1.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f816d1c8a0593375c289e285c96deaee9c2d8742cb0edbd26ee05588a9ae657", size = 36475 }, + { url = "https://files.pythonhosted.org/packages/ae/b1/954207a9bd259a57c8c71ba0a26739a32f0d6227529abc9412751f3a87d2/inflate64-1.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1facd35319b6a391ee4c3d709c7c650bcada8cd7141d86cd8c2257287f45e6e6", size = 35626 }, +] + [[package]] name = "iniconfig" version = "2.0.0" @@ -971,6 +1132,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/8e/9ad090d3553c280a8060fbf6e24dc1c0c29704ee7d1c372f0c174aa59285/matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca", size = 9899 }, ] +[[package]] +name = "multivolumefile" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/50/f0/a7786212b5a4cb9ba05ae84a2bbd11d1d0279523aea0424b6d981d652a14/multivolumefile-0.2.3.tar.gz", hash = "sha256:a0648d0aafbc96e59198d5c17e9acad7eb531abea51035d08ce8060dcad709d6", size = 77984 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/31/ec5f46fd4c83185b806aa9c736e228cb780f13990a9cf4da0beb70025fcc/multivolumefile-0.2.3-py3-none-any.whl", hash = "sha256:237f4353b60af1703087cf7725755a1f6fcaeeea48421e1896940cd1c920d678", size = 17037 }, +] + [[package]] name = "mypy-boto3-s3" version = "1.35.16" @@ -1083,6 +1253,7 @@ dependencies = [ { name = "gdown" }, { name = "gsheets" }, { name = "pandas" }, + { name = "py7zr" }, { name = "pyarrow" }, { name = "pydrive2" }, { name = "structlog" }, @@ -1114,6 +1285,7 @@ requires-dist = [ { name = "gdown", specifier = ">=4.5.2" }, { name = "gsheets", specifier = ">=0.6.1" }, { name = "pandas", specifier = ">=2.2.1" }, + { name = "py7zr", specifier = ">=0.22.0" }, { name = "pyarrow", specifier = ">=10.0.1" }, { name = "pydrive2", specifier = ">=1.15.0" }, { name = "structlog", specifier = ">=21.5.0" }, @@ -1132,7 +1304,7 @@ dev = [ { name = "pyright", specifier = "==1.1.373" }, { name = "pytest", specifier = ">=6.2.5" }, { name = "pytest-cov", specifier = ">=2.12.1" }, - { name = "ruff", specifier = "==0.1.6" }, + { name = "ruff", specifier = "==0.8.2" }, { name = "sphinx", specifier = "==4.5.0" }, { name = "watchdog", specifier = ">=2.1.5" }, ] @@ -1302,6 +1474,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842 }, ] +[[package]] +name = "py7zr" +version = "0.22.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "brotli", marker = "platform_python_implementation == 'CPython'" }, + { name = "brotlicffi", marker = "platform_python_implementation == 'PyPy'" }, + { name = "inflate64" }, + { name = "multivolumefile" }, + { name = "psutil", marker = "sys_platform != 'cygwin'" }, + { name = "pybcj" }, + { name = "pycryptodomex" }, + { name = "pyppmd" }, + { name = "pyzstd" }, + { name = "texttable" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/c3/0e05c711c16af0b9c47f3f77323303b338b9a871ba020d95d2b8dd6605ae/py7zr-0.22.0.tar.gz", hash = "sha256:c6c7aea5913535184003b73938490f9a4d8418598e533f9ca991d3b8e45a139e", size = 4992926 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/59/dd1750002c0f46099281116f8165247bc62dc85edad41cdd26e7b26de19d/py7zr-0.22.0-py3-none-any.whl", hash = "sha256:993b951b313500697d71113da2681386589b7b74f12e48ba13cc12beca79d078", size = 67906 }, +] + [[package]] name = "pyarrow" version = "17.0.0" @@ -1362,6 +1555,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/89/bc88a6711935ba795a679ea6ebee07e128050d6382eaa35a0a47c8032bdc/pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd", size = 181537 }, ] +[[package]] +name = "pybcj" +version = "1.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/d2/22e808b9d25ce3b43f5c8a9e22d873d403485ba55d84a4d6d5d044881762/pybcj-1.0.2.tar.gz", hash = "sha256:c7f5bef7f47723c53420e377bc64d2553843bee8bcac5f0ad076ab1524780018", size = 2111002 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/93/4735636b5905b7597068a2c7a10a8df0f668f28659207c274d64a4468b97/pybcj-1.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7bff28d97e47047d69a4ac6bf59adda738cf1d00adde8819117fdb65d966bdbc", size = 32556 }, + { url = "https://files.pythonhosted.org/packages/a6/37/443cd704397b6df54ff0822032e4815aca4e9badabc5ce1faac34235a40c/pybcj-1.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:198e0b4768b4025eb3309273d7e81dc53834b9a50092be6e0d9b3983cfd35c35", size = 23751 }, + { url = "https://files.pythonhosted.org/packages/9a/aa/5a19ed8661e979a4d3237a11706f9a16a474a2227fdd99ccb284be100a98/pybcj-1.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fa26415b4a118ea790de9d38f244312f2510a9bb5c65e560184d241a6f391a2d", size = 23980 }, + { url = "https://files.pythonhosted.org/packages/fe/5f/638ce03948905d267c8c0ccab81b8b4943a0324f63d8bdb0a0e2a85d4503/pybcj-1.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fabb2be57e4ca28ea36c13146cdf97d73abd27c51741923fc6ba1e8cd33e255c", size = 50155 }, + { url = "https://files.pythonhosted.org/packages/09/70/8b6a6cc2a5721f67f629bdc17875c0d603d57f360a19b099a7b4de19383d/pybcj-1.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75d6d613bae6f27678d5e44e89d61018779726aa6aa950c516d33a04b8af8c59", size = 49729 }, + { url = "https://files.pythonhosted.org/packages/89/06/2e41e34da0bb2adb3644cbf4366c344e5804a10f1153da7b3a23333f7db8/pybcj-1.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3ffae79ef8a1ea81ea2748ad7b7ad9b882aa88ddf65ce90f9e944df639eccc61", size = 54310 }, + { url = "https://files.pythonhosted.org/packages/b5/0f/de9e76c305d4dcd9d428a90ccac030f06c780bc30549fc449a944a6321bc/pybcj-1.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bdb4d8ff5cba3e0bd1adee7d20dbb2b4d80cb31ac04d6ea1cd06cfc02d2ecd0d", size = 53679 }, + { url = "https://files.pythonhosted.org/packages/1a/41/a807ff6b77ec8e49c749ed1d0db5649fbb1150c6fb5fb391115f4f1d743a/pybcj-1.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a29be917fbc99eca204b08407e0971e0205bfdad4b74ec915930675f352b669d", size = 24690 }, + { url = "https://files.pythonhosted.org/packages/27/0a/20bf70a7eb7c6b2668ff2af798254033c32a09d6c58ec9a87cd6aa843df5/pybcj-1.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a2562ebe5a0abec4da0229f8abb5e90ee97b178f19762eb925c1159be36828b3", size = 32581 }, + { url = "https://files.pythonhosted.org/packages/a9/b6/43977fe4296d2778c6dc67b596bb6a851eaea80f3dd4ff454e5fca8142c2/pybcj-1.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:af19bc61ded933001cd68f004ae2042bf1a78eb498a3c685ebd655fa1be90dbe", size = 23767 }, + { url = "https://files.pythonhosted.org/packages/89/c7/a61010f59406b8a45bb4865faa4b61d6b177dcfac04247fb56c7538d997d/pybcj-1.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f3f4a447800850aba7724a2274ea0a4800724520c1caf38f7d0dabf2f89a5e15", size = 23976 }, + { url = "https://files.pythonhosted.org/packages/10/7a/78848edbb6f12d9b86e375fc46135d9a204ededbf96682b05cb4b4fbd942/pybcj-1.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce1c8af7a4761d2b1b531864d84113948daa0c4245775c63bd9874cb955f4662", size = 51246 }, + { url = "https://files.pythonhosted.org/packages/9e/13/af86c86cdfb293e82dd0b6c4bbdf08645cd8993456ee3fb911c3eeed1b22/pybcj-1.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8007371f6f2b462f5aa05d5c2135d0a1bcf5b7bdd9bd15d86c730f588d10b7d3", size = 50754 }, + { url = "https://files.pythonhosted.org/packages/39/52/88600aa374b100612a1d82fca4b03eb4315e0084a05ee314ba1b771f7190/pybcj-1.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1079ca63ff8da5c936b76863690e0bd2489e8d4e0a3a340e032095dae805dd91", size = 55334 }, + { url = "https://files.pythonhosted.org/packages/56/67/3cf9747ef5b53e16a844217c6c9840be6289d05ec785500da2cc55cc25f2/pybcj-1.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e9a785eb26884429d9b9f6326e68c3638828c83bf6d42d2463c97ad5385caff2", size = 54714 }, + { url = "https://files.pythonhosted.org/packages/78/81/a71197903b503f54b85f4d352f909e701e9d26953577bd34d3fbe0520d5d/pybcj-1.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:9ea46e2d45469d13b7f25b08efcdb140220bab1ac5a850db0954591715b8caaa", size = 24693 }, + { url = "https://files.pythonhosted.org/packages/83/60/a3b43836895654aa93b5a8422adc3717359db98da9147abfabffef79f1e7/pybcj-1.0.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:21b5f2460629167340403d359289a173e0729ce8e84e3ce99462009d5d5e01a4", size = 32677 }, + { url = "https://files.pythonhosted.org/packages/50/b9/96c8d9577b0f5a701e4497408e6a331a08eb902aca8dfd4c5bb1eaab4779/pybcj-1.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:2940fb85730b9869254559c491cd83cf777e56c76a8a60df60e4be4f2a4248d7", size = 23813 }, + { url = "https://files.pythonhosted.org/packages/b7/1a/c80132feb084ec4098c0315a132799bddda8878113b5f956e21c4377f5f1/pybcj-1.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f40f3243139d675f43793a4e35c410c370f7b91ccae74e70c8b2f4877869f90e", size = 24019 }, + { url = "https://files.pythonhosted.org/packages/b1/94/62c3bf8a60b4787b46e21f43277d9cb8b6037c8ee183450f035a19a2bc4b/pybcj-1.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c2b3e60b65c7ac73e44335934e1e122da8d56db87840984601b3c5dc0ae4c19", size = 51927 }, + { url = "https://files.pythonhosted.org/packages/8b/9e/4ebd092251ef8d15408388be508617d5949cbba4baa2a6cfbb7e0a9b62c0/pybcj-1.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:746550dc7b5af4d04bb5fa4d065f18d39c925bcb5dee30db75747cd9a58bb6e8", size = 51665 }, + { url = "https://files.pythonhosted.org/packages/24/ea/da4637563468854bd361a69cd883946015f54fa119a5d9c655d26f151954/pybcj-1.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:8ce9b62b6aaa5b08773be8a919ecc4e865396c969f982b685eeca6e80c82abb7", size = 56041 }, + { url = "https://files.pythonhosted.org/packages/cf/b2/9b9e670818af925ed9a0168a5c021ccfcc089637d0e6651d16fd05896425/pybcj-1.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:493eab2b1f6f546730a6de0c5ceb75ce16f3767154e8ae30e2b70d41b928b7d2", size = 55606 }, + { url = "https://files.pythonhosted.org/packages/72/e9/d6b1bdf3a5aca8f3981145a5228ad51d72e2477a55927604a4768765e915/pybcj-1.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:ef55b96b7f2ed823e0b924de902065ec42ade856366c287dbb073fabd6b90ec1", size = 24719 }, + { url = "https://files.pythonhosted.org/packages/6e/18/ca43a186a570b3f6820a24f3ad726d0f0322f0b08f5550a92f99741a2e58/pybcj-1.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:fdb7cd8271471a5979d84915c1ee57eea7e0a69c893225fc418db66883b0e2a7", size = 32548 }, + { url = "https://files.pythonhosted.org/packages/f1/10/ee383b3450f7f13fbe234668fcf143b9a1e916e0cb84f5267ff3a5c6ad60/pybcj-1.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e96ae14062bdcddc3197300e6ee4efa6fbc6749be917db934eac66d0daaecb68", size = 23743 }, + { url = "https://files.pythonhosted.org/packages/1f/90/2d2851d694a7dc3640c6309af0e534cab145e7463bc408114db7eaa20115/pybcj-1.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a54ebdc8423ba99d75372708a882fcfc3b14d9d52cf195295ad53e5a47dab37f", size = 23971 }, + { url = "https://files.pythonhosted.org/packages/9a/29/8326dbfea26d643d5a95f836103ac278eb297143d881188d94b987e3a520/pybcj-1.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3602be737c6e9553c45ae89e6b0e556f64f34dabf27d5260317d1824d31b79d3", size = 49937 }, + { url = "https://files.pythonhosted.org/packages/ac/b2/26fa2cba6bc488380515929757cafbdbf01f30184a1aa11ef7ee35bb21a2/pybcj-1.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63dd2ca52a48841f561bfec0fa3f208d375b0a8dcd3d7b236459e683ae29221d", size = 49517 }, + { url = "https://files.pythonhosted.org/packages/58/7b/d5e39a73202eb7b67793a4313ae5a85bdbf1470899dc2d3119c6a2414e9b/pybcj-1.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8204a714029784b1a08a3d790430d80b423b68615c5b1e67aabca5bd5419b77d", size = 54141 }, + { url = "https://files.pythonhosted.org/packages/8e/f0/5abc858fe9d07338e485e86b2d82d1f1a0aa36c2af5271156c1140d04d15/pybcj-1.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:fde2376b180ae2620c102fbc3ef06638d306feae83964aaa5051ecbdda54845a", size = 53499 }, + { url = "https://files.pythonhosted.org/packages/9f/ac/4bad26429aab693235035f813dc60ff00ff5164acbbb98c4e26f190a21cd/pybcj-1.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:3b8d7810fb587adbffba025330cf212d9bbed8f29559656d05cb6609673f306a", size = 24686 }, +] + [[package]] name = "pycparser" version = "2.22" @@ -1371,6 +1604,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552 }, ] +[[package]] +name = "pycryptodomex" +version = "3.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/dc/e66551683ade663b5f07d7b3bc46434bf703491dbd22ee12d1f979ca828f/pycryptodomex-3.21.0.tar.gz", hash = "sha256:222d0bd05381dd25c32dd6065c071ebf084212ab79bab4599ba9e6a3e0009e6c", size = 4818543 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/5e/99f217d9881eead69607a2248dd7bbdf610837d7f5ad53f45a6cb71bbbfb/pycryptodomex-3.21.0-cp36-abi3-macosx_10_9_universal2.whl", hash = "sha256:34325b84c8b380675fd2320d0649cdcbc9cf1e0d1526edbe8fce43ed858cdc7e", size = 2499490 }, + { url = "https://files.pythonhosted.org/packages/ce/8f/4d0e2a859a6470289d64e39b419f01d2494dfa2e4995342d50f6c2834237/pycryptodomex-3.21.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:103c133d6cd832ae7266feb0a65b69e3a5e4dbbd6f3a3ae3211a557fd653f516", size = 1638037 }, + { url = "https://files.pythonhosted.org/packages/0c/9e/6e748c1fa814c956d356f93cf7192b19487ca56fc9e2a0bcde2bbc057601/pycryptodomex-3.21.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77ac2ea80bcb4b4e1c6a596734c775a1615d23e31794967416afc14852a639d3", size = 2172279 }, + { url = "https://files.pythonhosted.org/packages/46/3f/f5bef92b11750af9e3516d4e69736eeeff20a2818d34611508bef5a7b381/pycryptodomex-3.21.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9aa0cf13a1a1128b3e964dc667e5fe5c6235f7d7cfb0277213f0e2a783837cc2", size = 2258130 }, + { url = "https://files.pythonhosted.org/packages/de/4d/f0c65afd64ce435fd0547187ce6f99dfb37cdde16b05b57bca9f5c06966e/pycryptodomex-3.21.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:46eb1f0c8d309da63a2064c28de54e5e614ad17b7e2f88df0faef58ce192fc7b", size = 2297719 }, + { url = "https://files.pythonhosted.org/packages/1c/6a/2a1a101b0345ee70376ba93df8de6c8c01aac8341fda02970800873456a7/pycryptodomex-3.21.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:cc7e111e66c274b0df5f4efa679eb31e23c7545d702333dfd2df10ab02c2a2ce", size = 2164079 }, + { url = "https://files.pythonhosted.org/packages/3d/00/90a15f16c234815b660303c2d7266b41b401ea2605f3a90373e9d425e39f/pycryptodomex-3.21.0-cp36-abi3-musllinux_1_2_i686.whl", hash = "sha256:770d630a5c46605ec83393feaa73a9635a60e55b112e1fb0c3cea84c2897aa0a", size = 2333060 }, + { url = "https://files.pythonhosted.org/packages/61/74/49f5d20c514ccc631b940cc9dfec45dcce418dc84a98463a2e2ebec33904/pycryptodomex-3.21.0-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:52e23a0a6e61691134aa8c8beba89de420602541afaae70f66e16060fdcd677e", size = 2257982 }, + { url = "https://files.pythonhosted.org/packages/92/4b/d33ef74e2cc0025a259936661bb53432c5bbbadc561c5f2e023bcd73ce4c/pycryptodomex-3.21.0-cp36-abi3-win32.whl", hash = "sha256:a3d77919e6ff56d89aada1bd009b727b874d464cb0e2e3f00a49f7d2e709d76e", size = 1779052 }, + { url = "https://files.pythonhosted.org/packages/5b/be/7c991840af1184009fc86267160948350d1bf875f153c97bb471ad944e40/pycryptodomex-3.21.0-cp36-abi3-win_amd64.whl", hash = "sha256:b0e9765f93fe4890f39875e6c90c96cb341767833cfa767f41b490b506fa9ec0", size = 1816307 }, + { url = "https://files.pythonhosted.org/packages/af/ac/24125ad36778914a36f08d61ba5338cb9159382c638d9761ee19c8de822c/pycryptodomex-3.21.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:feaecdce4e5c0045e7a287de0c4351284391fe170729aa9182f6bd967631b3a8", size = 1694999 }, + { url = "https://files.pythonhosted.org/packages/93/73/be7a54a5903508070e5508925ba94493a1f326cfeecfff750e3eb250ea28/pycryptodomex-3.21.0-pp27-pypy_73-win32.whl", hash = "sha256:365aa5a66d52fd1f9e0530ea97f392c48c409c2f01ff8b9a39c73ed6f527d36c", size = 1769437 }, + { url = "https://files.pythonhosted.org/packages/e5/9f/39a6187f3986841fa6a9f35c6fdca5030ef73ff708b45a993813a51d7d10/pycryptodomex-3.21.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3efddfc50ac0ca143364042324046800c126a1d63816d532f2e19e6f2d8c0c31", size = 1619607 }, + { url = "https://files.pythonhosted.org/packages/f8/70/60bb08e9e9841b18d4669fb69d84b64ce900aacd7eb0ebebd4c7b9bdecd3/pycryptodomex-3.21.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0df2608682db8279a9ebbaf05a72f62a321433522ed0e499bc486a6889b96bf3", size = 1653571 }, + { url = "https://files.pythonhosted.org/packages/c9/6f/191b73509291c5ff0dddec9cc54797b1d73303c12b2e4017b24678e57099/pycryptodomex-3.21.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5823d03e904ea3e53aebd6799d6b8ec63b7675b5d2f4a4bd5e3adcb512d03b37", size = 1691548 }, + { url = "https://files.pythonhosted.org/packages/2d/c7/a0d3356f3074ac548afefa515ff46f3bea011deca607faf1c09b26dd5330/pycryptodomex-3.21.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:27e84eeff24250ffec32722334749ac2a57a5fd60332cd6a0680090e7c42877e", size = 1792099 }, + { url = "https://files.pythonhosted.org/packages/55/ee/9349856ee02826899fdc489016756865158217909a82dcc74cc4d55d33af/pycryptodomex-3.21.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:8ef436cdeea794015263853311f84c1ff0341b98fc7908e8a70595a68cefd971", size = 1619490 }, + { url = "https://files.pythonhosted.org/packages/07/93/e68fac121fcf761fd7a85a27f024c9238217e4d943c861a856ca354f412e/pycryptodomex-3.21.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a1058e6dfe827f4209c5cae466e67610bcd0d66f2f037465daa2a29d92d952b", size = 1653481 }, + { url = "https://files.pythonhosted.org/packages/b8/47/8a39243d09fd294c339c59834ba3c92715584f3ed0d92b6bacb26f803ce0/pycryptodomex-3.21.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9ba09a5b407cbb3bcb325221e346a140605714b5e880741dc9a1e9ecf1688d42", size = 1691442 }, + { url = "https://files.pythonhosted.org/packages/21/1c/f8860c558b44776573acd719c1e86fec14d42f29cf248eaba9c770151d14/pycryptodomex-3.21.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:8a9d8342cf22b74a746e3c6c9453cb0cfbb55943410e3a2619bd9164b48dc9d9", size = 1791966 }, +] + [[package]] name = "pydrive2" version = "1.20.0" @@ -1416,6 +1677,68 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/0c/0e3c05b1c87bb6a1c76d281b0f35e78d2d80ac91b5f8f524cebf77f51049/pyparsing-3.1.4-py3-none-any.whl", hash = "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c", size = 104100 }, ] +[[package]] +name = "pyppmd" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/39/c8/9039c7503577de08a3f4c81e7619583efdc16030da6d1a25268d3dca49c8/pyppmd-1.1.0.tar.gz", hash = "sha256:1d38ce2e4b7eb84b53bc8a52380b94f66ba6c39328b8800b30c2b5bf31693973", size = 1348949 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/10/b19621035862e2ae12a1ba14c5b5c0a0befb27906bc00691642d7bdbdce6/pyppmd-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c5cd428715413fe55abf79dc9fc54924ba7e518053e1fc0cbdf80d0d99cf1442", size = 75756 }, + { url = "https://files.pythonhosted.org/packages/85/4a/a7c172cd431c4e1ddf9be349dc4bcfea81c2a236d2fe51bbfdcd697af55a/pyppmd-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0e96cc43f44b7658be2ea764e7fa99c94cb89164dbb7cdf209178effc2168319", size = 47347 }, + { url = "https://files.pythonhosted.org/packages/0d/32/f7357e0412e977ede4d63ba8bf55d014e5ea5b311818b2b0a1fee6d91baa/pyppmd-1.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dd20142869094bceef5ab0b160f4fff790ad1f612313a1e3393a51fc3ba5d57e", size = 46640 }, + { url = "https://files.pythonhosted.org/packages/b5/8e/1f416819f0aab17de47b15b72d0e9b05e2bf795c6e28d9f403ac01398b74/pyppmd-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a4f9b51e45c11e805e74ea6f6355e98a6423b5bbd92f45aceee24761bdc3d3b8", size = 135666 }, + { url = "https://files.pythonhosted.org/packages/73/ac/7d07d3ac6874f235554de392de08e6a369001db43cd6a619af4fbe02fb55/pyppmd-1.1.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:459f85e928fb968d0e34fb6191fd8c4e710012d7d884fa2b317b2e11faac7c59", size = 132892 }, + { url = "https://files.pythonhosted.org/packages/09/76/61db4268a439cfba8736b14130d928d199633fab2360a2c5043332a427d2/pyppmd-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f73cf2aaf60477eef17f5497d14b6099d8be9748390ad2b83d1c88214d050c05", size = 138901 }, + { url = "https://files.pythonhosted.org/packages/8b/9c/546729489ae07c0d7c2bfe37c69ae1cd3ce35a18ab000480ea4e8f12754f/pyppmd-1.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2ea3ae0e92c0b5345cd3a4e145e01bbd79c2d95355481ea5d833b5c0cb202a2d", size = 139725 }, + { url = "https://files.pythonhosted.org/packages/f7/db/4e734e97541554a389e7adb2a2a5c86ad8ae35c4dafe817b12fdc317de1a/pyppmd-1.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:775172c740133c0162a01c1a5443d0e312246881cdd6834421b644d89a634b91", size = 131598 }, + { url = "https://files.pythonhosted.org/packages/b1/8f/530e47290e07d2fdedfd345fc72af08226ccdd4cc913c2b895a8396c17b6/pyppmd-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:14421030f1d46f69829698bdd960698a3b3df0925e3c470e82cfcdd4446b7bc1", size = 142767 }, + { url = "https://files.pythonhosted.org/packages/a5/f9/16e0adfef500b171a96ed3c95f4a4d999f99cc79de3e415146808b19c2fb/pyppmd-1.1.0-cp310-cp310-win32.whl", hash = "sha256:b691264f9962532aca3bba5be848b6370e596d0a2ca722c86df388be08d0568a", size = 41283 }, + { url = "https://files.pythonhosted.org/packages/37/8d/c4846ab632e13ead87189f31bcc51fc825c75078d162a4a9dc8aed0a5b97/pyppmd-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:216b0d969a3f06e35fbfef979706d987d105fcb1e37b0b1324f01ee143719c4a", size = 46078 }, + { url = "https://files.pythonhosted.org/packages/27/0e/9db5d7c6ca3159aa0f07c0f1d5c59079176e7c57740a61aca62a39661178/pyppmd-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1f8c51044ee4df1b004b10bf6b3c92f95ea86cfe1111210d303dca44a56e4282", size = 75781 }, + { url = "https://files.pythonhosted.org/packages/f0/1b/4894b5c71feee76d3dfccf4383b59841f9bfd27aecf912b6542a2ab1e073/pyppmd-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ac25b3a13d1ac9b8f0bde46952e10848adc79d932f2b548a6491ef8825ae0045", size = 47370 }, + { url = "https://files.pythonhosted.org/packages/50/98/57b2c281e546f682279bd4a2577045d1f6d527c8fa2151a990b2a9bc48c2/pyppmd-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c8d3003eebe6aabe22ba744a38a146ed58a25633420d5da882b049342b7c8036", size = 46633 }, + { url = "https://files.pythonhosted.org/packages/06/72/b7e37aa69b7a105bcc119bc171437fbcb104aef2568b68ec8ed21a3fcdd1/pyppmd-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c520656bc12100aa6388df27dd7ac738577f38bf43f4a4bea78e1861e579ea5", size = 138233 }, + { url = "https://files.pythonhosted.org/packages/60/73/4f53a3c7730e1cba3f210b35ed6779e0fe302739196f43452664e079c0b5/pyppmd-1.1.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c2a3e807028159a705951f5cb5d005f94caed11d0984e59cc50506de543e22d", size = 135486 }, + { url = "https://files.pythonhosted.org/packages/31/7c/956ebf1f07506bb59e6f13ef068d91f1bec828758d399b455b175b668f6c/pyppmd-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec8a2447e69444703e2b273247bfcd4b540ec601780eff07da16344c62d2993d", size = 141183 }, + { url = "https://files.pythonhosted.org/packages/73/b4/4863499e012c555f4619dbebc5b83d79818e0161d9b6fb8b1e709fb1d6c7/pyppmd-1.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b9e0c8053e69cad6a92a0889b3324f567afc75475b4f54727de553ac4fc85780", size = 141752 }, + { url = "https://files.pythonhosted.org/packages/b4/cc/44e175222b31f86d0192d1d0d2c46c4bf0e933c9a06a65ff39596ad05666/pyppmd-1.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:5938d256e8d2a2853dc3af8bb58ae6b4a775c46fc891dbe1826a0b3ceb624031", size = 133921 }, + { url = "https://files.pythonhosted.org/packages/f1/d9/2f2e222d43ab274909e8dcd16d25cd4cc0245a8d59f93f8d6397cd4dc49f/pyppmd-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1ce5822d8bea920856232ccfb3c26b56b28b6846ea1b0eb3d5cb9592a026649e", size = 145191 }, + { url = "https://files.pythonhosted.org/packages/6d/e7/1214571442624e2314ed1ed5ba0081358335fc760fb455c3d8df83b118c6/pyppmd-1.1.0-cp311-cp311-win32.whl", hash = "sha256:2a9e894750f2a52b03e3bc0d7cf004d96c3475a59b1af7e797d808d7d29c9ffe", size = 41286 }, + { url = "https://files.pythonhosted.org/packages/8e/7f/d3cc8443bd2b56bc54ea205dcf73d70ef8d4342096ff33fc8719956f45e9/pyppmd-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:969555c72e72fe2b4dd944127521a8f2211caddb5df452bbc2506b5adfac539e", size = 46087 }, + { url = "https://files.pythonhosted.org/packages/bf/0b/4c8e3a92c4366a9aa2d801ab4bd7ba72bd1d214da890dd91ab4d73e52878/pyppmd-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9d6ef8fd818884e914bc209f7961c9400a4da50d178bba25efcef89f09ec9169", size = 76116 }, + { url = "https://files.pythonhosted.org/packages/e1/0b/45fdf5a28c810ed4d3c0cb05ae5346e2972cdbfe89f374b263e07c5b820d/pyppmd-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:95f28e2ecf3a9656bd7e766aaa1162b6872b575627f18715f8b046e8617c124a", size = 47633 }, + { url = "https://files.pythonhosted.org/packages/56/a4/4aa1d36d98f3786c8b12ac96ac8234d7dc3c2a9e8f5174a5698f424099ec/pyppmd-1.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:37f3557ea65ee417abcdf5f49d35df00bb9f6f252639cae57aeefcd0dd596133", size = 46704 }, + { url = "https://files.pythonhosted.org/packages/d9/70/a49389a6666f670db5ecc7caa37030c9a9abfeea455c387172584551a271/pyppmd-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e84b25d088d7727d50218f57f92127cdb839acd6ec3de670b6680a4cf0b2d2a", size = 139145 }, + { url = "https://files.pythonhosted.org/packages/30/4c/f08cdf618744a3cce0da106ecf6e427b24d27b0bb1484afc40b88ca23a39/pyppmd-1.1.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:99ed42891986dac8c2ecf52bddfb777900233d867aa18849dbba6f3335600466", size = 136618 }, + { url = "https://files.pythonhosted.org/packages/bb/e0/afc0fb971c893e9e72cc8d70df93c50b3f3ebb12b4bdb21f869b775faf7e/pyppmd-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6fe69b82634488ada75ba07efb90cd5866fa3d64a2c12932b6e8ae207a14e5f", size = 142757 }, + { url = "https://files.pythonhosted.org/packages/26/b2/793e92c7a66de0b0b8d777c3c4df3ee5a5bec7fbaf0b69ab7374cefefa43/pyppmd-1.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:60981ffde1fe6ade750b690b35318c41a1160a8505597fda2c39a74409671217", size = 142749 }, + { url = "https://files.pythonhosted.org/packages/5e/6e/a1bf750bc7ed025a06600c65917d02e3c6dea7dfa728746c7251d4910d37/pyppmd-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:46e8240315476f57aac23d71e6de003e122b65feba7c68f4cc46a089a82a7cd4", size = 135033 }, + { url = "https://files.pythonhosted.org/packages/1e/ee/4a12a4b1990f1fabb77f9ef94d2cd6c795690eec79ad135b8236dc59dbd2/pyppmd-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c0308e2e76ecb4c878a18c2d7a7c61dbca89b4ef138f65d5f5ead139154dcdea", size = 146510 }, + { url = "https://files.pythonhosted.org/packages/04/cd/a6571420345315f5340ac10897726303ae07260cb025dc4a60371d1e8b97/pyppmd-1.1.0-cp312-cp312-win32.whl", hash = "sha256:b4fa4c27dc1314d019d921f2aa19e17f99250557e7569eeb70e180558f46af74", size = 41332 }, + { url = "https://files.pythonhosted.org/packages/c0/a4/af77129d671d6adcc6c82e1b0f03f0ad0b70c44ac70ed4c72b5c8952553b/pyppmd-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:c269d21e15f4175df27cf00296476097af76941f948734c642d7fb6e85b9b3b9", size = 46193 }, + { url = "https://files.pythonhosted.org/packages/b4/10/144f811290a36d6cf5b5c8ae9b68533abe4bea160285be73435d55c361e1/pyppmd-1.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2aeea1bf585c6b8771fa43a6abd704da92f8a46a6d0020953af15d7f3c82e48c", size = 75760 }, + { url = "https://files.pythonhosted.org/packages/dc/2b/242b6ba7938e77f14dadc0e5d638288b5588f0aff5e5d2c0428726606e5e/pyppmd-1.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7759bdb137694d4ab0cfa5ff2c75c212d90714c7da93544694f68001a0c38e12", size = 47339 }, + { url = "https://files.pythonhosted.org/packages/ec/80/91a13a5d0da916e7243f66839941976b2729bac1d3ca1737c20f1d59b216/pyppmd-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:db64a4fe956a2e700a737a1d019f526e6ccece217c163b28b354a43464cc495b", size = 46636 }, + { url = "https://files.pythonhosted.org/packages/aa/18/f6126af21186eee49f9aa090c36acc9d2bccef4c7d077d23b2f24dfb804c/pyppmd-1.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f788ae8f5a9e79cd777b7969d3401b2a2b87f47abe306c2a03baca30595e9bd", size = 135484 }, + { url = "https://files.pythonhosted.org/packages/03/55/ebfeb5d1085f8a2bd03f498aa7e6ef9635380bf1f88badd1f3b944198ada/pyppmd-1.1.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:324a178935c140210fca2043c688b77e79281da8172d2379a06e094f41735851", size = 132741 }, + { url = "https://files.pythonhosted.org/packages/df/92/f0a7a6e372c4bd659b5528ff179676522aa72bd8c7a071e757a490ff988e/pyppmd-1.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:363030bbcb7902fb9eeb59ffc262581ca5dd7790ba950328242fd2491c54d99b", size = 138761 }, + { url = "https://files.pythonhosted.org/packages/43/32/0f32a70ef3fbe287dc53fd86408e9c2f60515ab356bd728bde9fcebcb598/pyppmd-1.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:31b882584f86440b0ff7906385c9f9d9853e5799197abaafdae2245f87d03f01", size = 139568 }, + { url = "https://files.pythonhosted.org/packages/fd/02/730882ea61653af4ef22b3621e9a03e85e61c90884fb47df4495706439f5/pyppmd-1.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b991b4501492ec3380b605fe30bee0b61480d305e98519d81c2a658b2de01593", size = 131441 }, + { url = "https://files.pythonhosted.org/packages/9e/cd/624b6582766b8e60c4356ebcfe73504c81d1395f36d0d409ff081be56e3d/pyppmd-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b6108044d943b826f97a9e79201242f61392d6c1fadba463b2069c4e6bc961e1", size = 142558 }, + { url = "https://files.pythonhosted.org/packages/65/46/1908b1ef3ba6f1450bc1a3e45a6b39cfe4b33456a312d1add0041ba1bbe4/pyppmd-1.1.0-cp39-cp39-win32.whl", hash = "sha256:c45ce2968b7762d2cacf622b0a8f260295c6444e0883fd21a21017e3eaef16ed", size = 41284 }, + { url = "https://files.pythonhosted.org/packages/63/29/081b03f989deb7ce2f70461dac12ecc422e9abef6b7b7a1933945c96b06f/pyppmd-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:f5289f32ab4ec5f96a95da51309abd1769f928b0bff62047b3bc25c878c16ccb", size = 46071 }, + { url = "https://files.pythonhosted.org/packages/6a/e2/1d5fbd6dde1234b635000072c8d1d87c7ed3acf01a3c4aa8082504d58bc5/pyppmd-1.1.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ad5da9f7592158e6b6b51d7cd15e536d8b23afbb4d22cba4e5744c7e0a3548b1", size = 41505 }, + { url = "https://files.pythonhosted.org/packages/24/66/9215c5dda61b3aa3259902a586dacd198b4b0793ab99228734091b5e7fa7/pyppmd-1.1.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc6543e7d12ef0a1466d291d655e3d6bca59c7336dbb53b62ccdd407822fb52b", size = 44814 }, + { url = "https://files.pythonhosted.org/packages/1a/87/cc2aa429688f238ae30f26b8334194a21e25643d3257c9e5b14cccdc578e/pyppmd-1.1.0-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a5e4008a45910e3c8c227f6f240de67eb14454c015dc3d8060fc41e230f395d3", size = 43629 }, + { url = "https://files.pythonhosted.org/packages/9f/96/cd3f64f6bdce091ffb6d2c1c23dc91e8b94e312a5d08cd648625555fb69e/pyppmd-1.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9301fa39d1fb0ed09a10b4c5d7f0074113e96a1ead16ba7310bedf95f7ef660c", size = 43911 }, + { url = "https://files.pythonhosted.org/packages/e6/ab/02ab90e2dddf2dd55e30e64fa0509627c6e0c86b26503a6df95ae55b1e45/pyppmd-1.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:59521a3c6028da0cb5780ba16880047b00163432a6b975da2f6123adfc1b0be8", size = 42427 }, + { url = "https://files.pythonhosted.org/packages/fa/0e/05db05c0da6a9bbb1f32de107b9f92e95ca9cb407c2082c7a0bee0a8868b/pyppmd-1.1.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:cce8cd2d4ceebe2dbf41db6dfebe4c2e621314b3af8a2df2cba5eb5fa277f122", size = 41500 }, + { url = "https://files.pythonhosted.org/packages/ba/89/48b01bcab274c59548897de336b807777b8e4abbd0465ed37c04152d13e2/pyppmd-1.1.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62e57927dbcb91fb6290a41cd83743b91b9d85858efb16a0dd34fac208ee1c6b", size = 44810 }, + { url = "https://files.pythonhosted.org/packages/a9/e4/bf4aa2305eda1bdf530d8127e07dd324ae923e664db3aa7bf153f1d749ec/pyppmd-1.1.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:435317949a6f35e54cdf08e0af6916ace427351e7664ac1593980114668f0aaa", size = 43625 }, + { url = "https://files.pythonhosted.org/packages/fc/be/33478c13fad90049c35034d9f7de658353f82fc2cd91c57efd904b71ebb7/pyppmd-1.1.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f66b0d0e32b8fb8707f1d2552f13edfc2917e8ed0bdf4d62e2ce190d2c70834", size = 43909 }, + { url = "https://files.pythonhosted.org/packages/60/34/922d8ca6879f08d17f9771e6ef65e9491c2ebbd48934997a3ff01285e55e/pyppmd-1.1.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:650a663a591e06fb8096c213f4070b158981c8c3bf9c166ce7e4c360873f2750", size = 42422 }, +] + [[package]] name = "pyright" version = "1.1.373" @@ -1648,6 +1971,96 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ca/63/27e6142b4f67a442ee480986ca5b88edb01462dd2319843057683a5148bd/pyzmq-26.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:4f78c88905461a9203eac9faac157a2a0dbba84a0fd09fd29315db27be40af9f", size = 550757 }, ] +[[package]] +name = "pyzstd" +version = "0.16.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/62/14/878fee4072cecb1cc6e061c7d0d933e481389c27de939538c9cc3f18894a/pyzstd-0.16.2.tar.gz", hash = "sha256:179c1a2ea1565abf09c5f2fd72f9ce7c54b2764cf7369e05c0bfd8f1f67f63d2", size = 789505 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/a9/efad061c5a982f859ba8bf5de565d73567f87ad8bba3364fe28e9a8672b6/pyzstd-0.16.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:637376c8f8cbd0afe1cab613f8c75fd502bd1016bf79d10760a2d5a00905fe62", size = 372191 }, + { url = "https://files.pythonhosted.org/packages/b6/36/eb6dcfacb273ca13dfa20d296f27ffd0a6c53677965f868625edf764b71e/pyzstd-0.16.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3e7a7118cbcfa90ca2ddbf9890c7cb582052a9a8cf2b7e2c1bbaf544bee0f16a", size = 295083 }, + { url = "https://files.pythonhosted.org/packages/fb/76/a7862487402123f221439808ed50915e00cfc8e1df7365af366610176347/pyzstd-0.16.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a74cb1ba05876179525144511eed3bd5a509b0ab2b10632c1215a85db0834dfd", size = 390166 }, + { url = "https://files.pythonhosted.org/packages/b8/52/1e1ab63026d67f18b9841285576d59bb799b838a5de4f852ad9e054674a1/pyzstd-0.16.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c084dde218ffbf112e507e72cbf626b8f58ce9eb23eec129809e31037984662", size = 472043 }, + { url = "https://files.pythonhosted.org/packages/0d/24/14c8948b9d16d399ff80504bc404bb091b0eb5339f6fbdad0481da751c09/pyzstd-0.16.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4646459ebd3d7a59ddbe9312f020bcf7cdd1f059a2ea07051258f7af87a0b31", size = 415258 }, + { url = "https://files.pythonhosted.org/packages/6b/3e/e4c7f449af9d19975ff5d333a58330317cf8b05fe4754106c694a29e7c25/pyzstd-0.16.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14bfc2833cc16d7657fc93259edeeaa793286e5031b86ca5dc861ba49b435fce", size = 413680 }, + { url = "https://files.pythonhosted.org/packages/10/09/8918853028cf593c141456b9a42d68420beec3f16a8cc4f1aa5d0b8b0c84/pyzstd-0.16.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f27d488f19e5bf27d1e8aa1ae72c6c0a910f1e1ffbdf3c763d02ab781295dd27", size = 412630 }, + { url = "https://files.pythonhosted.org/packages/47/20/5a4c899530571e0e8ecdcb9dc7e3fc38491d4b342fbd7d8413805c88013b/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:91e134ca968ff7dcfa8b7d433318f01d309b74ee87e0d2bcadc117c08e1c80db", size = 404980 }, + { url = "https://files.pythonhosted.org/packages/0a/1d/aeeeebb702d3500a01b5b1029ba1716aea3afa75e8aacb904806b3f1afe5/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:6b5f64cd3963c58b8f886eb6139bb8d164b42a74f8a1bb95d49b4804f4592d61", size = 418000 }, + { url = "https://files.pythonhosted.org/packages/fc/0c/66ca36d24ad97af40a8fe8de9e3f316a5f4fd2fb3cab8634a2f7da5571c8/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:0b4a8266871b9e0407f9fd8e8d077c3558cf124d174e6357b523d14f76971009", size = 485576 }, + { url = "https://files.pythonhosted.org/packages/39/66/6c1de1347de94aa85f60e854cccae0948bda2eda2351e4d47c8bb0a7cf18/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:1bb19f7acac30727354c25125922aa59f44d82e0e6a751df17d0d93ff6a73853", size = 564542 }, + { url = "https://files.pythonhosted.org/packages/6d/46/75365a3ab279d58e69d410ce0a21527e689fa651837227e23dee294d096f/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3008325b7368e794d66d4d98f2ee1d867ef5afd09fd388646ae02b25343c420d", size = 430619 }, + { url = "https://files.pythonhosted.org/packages/0d/62/17bf81d42acbd39bffdea559b6fbd7ec331cd74bc52f249e536fefe5480d/pyzstd-0.16.2-cp310-cp310-win32.whl", hash = "sha256:66f2d5c0bbf5bf32c577aa006197b3525b80b59804450e2c32fbcc2d16e850fd", size = 218224 }, + { url = "https://files.pythonhosted.org/packages/f7/b6/281245890df08a567186c6e262c43d68581291cca107c8d7304c37708e46/pyzstd-0.16.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fe5f5459ebe1161095baa7a86d04ab625b35148f6c425df0347ed6c90a2fd58", size = 245012 }, + { url = "https://files.pythonhosted.org/packages/10/5a/19d7aec81853f6dc53eabad388227e3beecfaca4788af23b8807a0ea2112/pyzstd-0.16.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1bdbe7f01c7f37d5cd07be70e32a84010d7dfd6677920c0de04cf7d245b60d", size = 372192 }, + { url = "https://files.pythonhosted.org/packages/29/35/2eb025e6a0fff49b5de8bea20e82e4d7d5456e634bf3809123fbe5e5f194/pyzstd-0.16.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1882a3ceaaf9adc12212d587d150ec5e58cfa9a765463d803d739abbd3ac0f7a", size = 295084 }, + { url = "https://files.pythonhosted.org/packages/04/1f/03785d7ff1ce73b9347533f798cb27afa57768e66012f97b18b7b7303158/pyzstd-0.16.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea46a8b9d60f6a6eba29facba54c0f0d70328586f7ef0da6f57edf7e43db0303", size = 390167 }, + { url = "https://files.pythonhosted.org/packages/b7/59/e307622115a2df30075efbd28933dc0ad8f2007c5ba5a3eb49c956de3d56/pyzstd-0.16.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d7865bc06589cdcecdede0deefe3da07809d5b7ad9044c224d7b2a0867256957", size = 472038 }, + { url = "https://files.pythonhosted.org/packages/97/21/870fda5454240089e9c37625320580d392b03beaeae4889c67c0a21c4d34/pyzstd-0.16.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:52f938a65b409c02eb825e8c77fc5ea54508b8fc44b5ce226db03011691ae8cc", size = 415217 }, + { url = "https://files.pythonhosted.org/packages/3c/35/b33faeeb9c96fddd08bf7871c9f5c4638c32ad79227155922fd4a63190c5/pyzstd-0.16.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e97620d3f53a0282947304189deef7ca7f7d0d6dfe15033469dc1c33e779d5e5", size = 413714 }, + { url = "https://files.pythonhosted.org/packages/aa/a3/b9058dd43eb52025a2ca78946dcb9ef9d8984acac172a698bcf12712217c/pyzstd-0.16.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7c40e9983d017108670dc8df68ceef14c7c1cf2d19239213274783041d0e64c", size = 412568 }, + { url = "https://files.pythonhosted.org/packages/12/31/fe7d462c912f2040775bfa2af4327f9fcebb16e8fa9c3bfa058bc1306722/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7cd4b3b2c6161066e4bde6af1cf78ed3acf5d731884dd13fdf31f1db10830080", size = 404988 }, + { url = "https://files.pythonhosted.org/packages/48/4c/582aca0e5210436499bce1639a8d15da3f76f8d5827da1aa3eeb2c4e271c/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:454f31fd84175bb203c8c424f2255a343fa9bd103461a38d1bf50487c3b89508", size = 417961 }, + { url = "https://files.pythonhosted.org/packages/39/e9/54f53641ff10b4ea18d3ba159b03bd07e6ae5a5b7ae01f1329b0c35b8ca2/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:5ef754a93743f08fb0386ce3596780bfba829311b49c8f4107af1a4bcc16935d", size = 485587 }, + { url = "https://files.pythonhosted.org/packages/ce/65/25243b3fea9e52a20bfece1b12e3d3ee3125f17b1735aab08cb9a7a760b4/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:be81081db9166e10846934f0e3576a263cbe18d81eca06e6a5c23533f8ce0dc6", size = 564543 }, + { url = "https://files.pythonhosted.org/packages/3b/3c/324b8ddca55b4b073b413cea3e0587af3c8153ccf7d6d63ed294831f2095/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:738bcb2fa1e5f1868986f5030955e64de53157fa1141d01f3a4daf07a1aaf644", size = 430628 }, + { url = "https://files.pythonhosted.org/packages/db/a1/aca18925e23bceb833fc742ebaf87aa9d1ba8b178f0332bd108fc8966482/pyzstd-0.16.2-cp311-cp311-win32.whl", hash = "sha256:0ea214c9b97046867d1657d55979021028d583704b30c481a9c165191b08d707", size = 218215 }, + { url = "https://files.pythonhosted.org/packages/c0/7f/0f5d1d1891e6c6e14d846d2881a06ab7e5e97cabeb5e1e9e53debec4091a/pyzstd-0.16.2-cp311-cp311-win_amd64.whl", hash = "sha256:c17c0fc02f0e75b0c7cd21f8eaf4c6ce4112333b447d93da1773a5f705b2c178", size = 245055 }, + { url = "https://files.pythonhosted.org/packages/28/15/20046759d138733e7150afa6aa15f322022d7587968e2dbd5b36fbf8aa86/pyzstd-0.16.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d4081fd841a9efe9ded7290ee7502dbf042c4158b90edfadea3b8a072c8ec4e1", size = 373230 }, + { url = "https://files.pythonhosted.org/packages/51/8d/55b536edaecf19d2f8dbd8fbaefd184f2f9cc6b71d241caa6d86bed96813/pyzstd-0.16.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fd3fa45d2aeb65367dd702806b2e779d13f1a3fa2d13d5ec777cfd09de6822de", size = 295699 }, + { url = "https://files.pythonhosted.org/packages/11/14/086e7f690154c6f3d9bdb46da26a4cd3c9e0b284346ce10943711ca48c32/pyzstd-0.16.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8b5f0d2c07994a5180d8259d51df6227a57098774bb0618423d7eb4a7303467", size = 390556 }, + { url = "https://files.pythonhosted.org/packages/90/d2/c6d854705d6fa0ad876209b4ba796ab31d85b710d1459029f2cb41085a8d/pyzstd-0.16.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60c9d25b15c7ae06ed5d516d096a0d8254f9bed4368b370a09cccf191eaab5cb", size = 472928 }, + { url = "https://files.pythonhosted.org/packages/aa/38/f97dd871e446adc834349caa605dbaf5bac86763a255f62c809cc2459c85/pyzstd-0.16.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:29acf31ce37254f6cad08deb24b9d9ba954f426fa08f8fae4ab4fdc51a03f4ae", size = 416057 }, + { url = "https://files.pythonhosted.org/packages/53/be/0c5ad7bf29dc890f6a3303760b9802aeeafa4e3ffb598de625f501986bfe/pyzstd-0.16.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ec77612a17697a9f7cf6634ffcee616eba9b997712fdd896e77fd19ab3a0618", size = 414613 }, + { url = "https://files.pythonhosted.org/packages/1f/1a/d3a1edcd59e2f62a35ac6257d2b86a2c872ae9a8e925380620a8db0d9a9a/pyzstd-0.16.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:313ea4974be93be12c9a640ab40f0fc50a023178aae004a8901507b74f190173", size = 413236 }, + { url = "https://files.pythonhosted.org/packages/f2/8d/912430c2310466c14a89a5a529b72eddef7e73fa733806dbe0b030cf3495/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e91acdefc8c2c6c3b8d5b1b5fe837dce4e591ecb7c0a2a50186f552e57d11203", size = 405536 }, + { url = "https://files.pythonhosted.org/packages/9e/83/4edb419a13b9d1e1debc01e88084eba93a5f7c10ef198da11f6782857c73/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:929bd91a403539e72b5b5cb97f725ac4acafe692ccf52f075e20cd9bf6e5493d", size = 419145 }, + { url = "https://files.pythonhosted.org/packages/8f/e9/62a169eddc37aefac480ee3b3318c221f6731e1e342dafd9e05b7fdaa7c5/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:740837a379aa32d110911ebcbbc524f9a9b145355737527543a884bd8777ca4f", size = 487157 }, + { url = "https://files.pythonhosted.org/packages/57/9d/5949f2a0144d1f99fab7914f854b582d2784c73139cc190e603e4d6b7b37/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:adfc0e80dd157e6d1e0b0112c8ecc4b58a7a23760bd9623d74122ef637cfbdb6", size = 565918 }, + { url = "https://files.pythonhosted.org/packages/de/ce/647b9c7602ac477c9e62cf9399810f72bb5dba8f508e7cdf8be1d260e6f9/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:79b183beae1c080ad3dca39019e49b7785391947f9aab68893ad85d27828c6e7", size = 431373 }, + { url = "https://files.pythonhosted.org/packages/8b/fb/4141e3d4549eea26e5a59ec723eade271980816cb2ed7613df855baa672f/pyzstd-0.16.2-cp312-cp312-win32.whl", hash = "sha256:b8d00631a3c466bc313847fab2a01f6b73b3165de0886fb03210e08567ae3a89", size = 218541 }, + { url = "https://files.pythonhosted.org/packages/51/b9/e1373b179129c2095d70bd1df02a51d388f4c7e4ecb62acb4e5e9570269b/pyzstd-0.16.2-cp312-cp312-win_amd64.whl", hash = "sha256:c0d43764e9a60607f35d8cb3e60df772a678935ab0e02e2804d4147377f4942c", size = 245320 }, + { url = "https://files.pythonhosted.org/packages/66/10/cc7c764c7673f1af1728abdcf58e58f88ef5d44ab4500677a2b7b4c01e7d/pyzstd-0.16.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3ae9ae7ad730562810912d7ecaf1fff5eaf4c726f4b4dfe04784ed5f06d7b91f", size = 373223 }, + { url = "https://files.pythonhosted.org/packages/3f/a7/bcaf7d635ee929dd4d08ae1c35101892db56a11542471eecfbf46b9dd988/pyzstd-0.16.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2ce8d3c213f76a564420f3d0137066ac007ce9fb4e156b989835caef12b367a7", size = 295701 }, + { url = "https://files.pythonhosted.org/packages/93/49/a604113a2f3135b29371a894c0faad22d7ea3f7b58f38d77baad8a817483/pyzstd-0.16.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c2c14dac23c865e2d78cebd9087e148674b7154f633afd4709b4cd1520b99a61", size = 392395 }, + { url = "https://files.pythonhosted.org/packages/b0/38/886ecf3ebb13a4b6e3ee85f448f54eef37a5ae2b453bd9d5d9edc909e119/pyzstd-0.16.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4527969d66a943e36ef374eda847e918077de032d58b5df84d98ffd717b6fa77", size = 474523 }, + { url = "https://files.pythonhosted.org/packages/14/98/121da6ac072c00090c218b4888ef00ead15979f09a657d9a5ff770d6bb17/pyzstd-0.16.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd8256149b88e657e99f31e6d4b114c8ff2935951f1d8bb8e1fe501b224999c0", size = 417974 }, + { url = "https://files.pythonhosted.org/packages/b6/ba/56652a67c0bcfaceb2945e5f07d5aa21af86e07cf33d1ae47bb3529a56c3/pyzstd-0.16.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5bd1f1822d65c9054bf36d35307bf8ed4aa2d2d6827431761a813628ff671b1d", size = 414587 }, + { url = "https://files.pythonhosted.org/packages/cc/30/cab6f45101f0113ced609ef65482aedd276e0f022d9f25a327d4284142f5/pyzstd-0.16.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6733f4d373ec9ad2c1976cf06f973a3324c1f9abe236d114d6bb91165a397d", size = 415071 }, + { url = "https://files.pythonhosted.org/packages/6d/44/2187fc8a46662926943aeb16d639dd4f3d06267c7e8abb2c6f97700ab11c/pyzstd-0.16.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7bec165ab6524663f00b69bfefd13a46a69fed3015754abaf81b103ec73d92c6", size = 407835 }, + { url = "https://files.pythonhosted.org/packages/de/d5/6edca97d5453cba820d2ad5630e6ec1fcfad66f69af5ad7d6c688ea301be/pyzstd-0.16.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e4460fa6949aac6528a1ad0de8871079600b12b3ef4db49316306786a3598321", size = 421755 }, + { url = "https://files.pythonhosted.org/packages/54/c1/1a0339e014ed97f4e6fd9166b0409ceda8f32e28e8ecda70fd7bb0915566/pyzstd-0.16.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:75df79ea0315c97d88337953a17daa44023dbf6389f8151903d371513f503e3c", size = 489174 }, + { url = "https://files.pythonhosted.org/packages/07/01/c65f2c9f0b902b33efcb0bdf3cbd07fc828fda6ff6333189eb71cf7acc60/pyzstd-0.16.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:93e1d45f4a196afb6f18682c79bdd5399277ead105b67f30b35c04c207966071", size = 573025 }, + { url = "https://files.pythonhosted.org/packages/a7/54/7ab9cc54171b7f8bb97cfd1c1aa7fcb706a4babeb629732529d8111bc4e6/pyzstd-0.16.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:075e18b871f38a503b5d23e40a661adfc750bd4bd0bb8b208c1e290f3ceb8fa2", size = 429582 }, + { url = "https://files.pythonhosted.org/packages/6c/a5/f9c950bb378dd1335bc4cc56444ec2ab40b1dab085c5798c5d16a9bf9d0b/pyzstd-0.16.2-cp313-cp313-win32.whl", hash = "sha256:9e4295eb299f8d87e3487852bca033d30332033272a801ca8130e934475e07a9", size = 218544 }, + { url = "https://files.pythonhosted.org/packages/9a/df/a15b9a8a59cd9908ae2b70bce2cb4ac3e2d7da11414ee0d0ceb46e4d0439/pyzstd-0.16.2-cp313-cp313-win_amd64.whl", hash = "sha256:18deedc70f858f4cf574e59f305d2a0678e54db2751a33dba9f481f91bc71c28", size = 245313 }, + { url = "https://files.pythonhosted.org/packages/e0/38/43002103a545bc953e532973596e905550e9626973c1b282e04e01038ac6/pyzstd-0.16.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a9892b707ef52f599098b1e9528df0e7849c5ec01d3e8035fb0e67de4b464839", size = 372192 }, + { url = "https://files.pythonhosted.org/packages/61/be/28dfeba9dbad8ed19d6aefa0d6623d1ee97e83c6c1e97910439428655f28/pyzstd-0.16.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4fbd647864341f3c174c4a6d7f20e6ea6b4be9d840fb900dc0faf0849561badc", size = 295080 }, + { url = "https://files.pythonhosted.org/packages/63/c2/c7e5244f2dde72df3fb2b7b952e8d01bac20cd78dc0d585d0a060ca565b0/pyzstd-0.16.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20ac2c15656cc6194c4fed1cb0e8159f9394d4ea1d58be755448743d2ec6c9c4", size = 390165 }, + { url = "https://files.pythonhosted.org/packages/ff/30/52560cb88179fa3ff7536429c0d7b83aeecea86ecb2d180a4afc991502e5/pyzstd-0.16.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b239fb9a20c1be3374b9a2bd183ba624fd22ad7a3f67738c0d80cda68b4ae1d3", size = 472040 }, + { url = "https://files.pythonhosted.org/packages/69/a7/ab1e19626da5a8ff58493d6928d9d0da4931034e7a124949bf1a1705daaf/pyzstd-0.16.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cc52400412cdae2635e0978b8d6bcc0028cc638fdab2fd301f6d157675d26896", size = 415255 }, + { url = "https://files.pythonhosted.org/packages/28/0d/bf7c9388fe43c7051a2ced4645e58a493a35c62e68307b5aaf0fb129b008/pyzstd-0.16.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b766a6aeb8dbb6c46e622e7a1aebfa9ab03838528273796941005a5ce7257b1", size = 413679 }, + { url = "https://files.pythonhosted.org/packages/58/2a/1e0738740a8bd2b1f4a74be86297c5776936b66b3a5340d8e4ae84c5844f/pyzstd-0.16.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd4b8676052f9d59579242bf3cfe5fd02532b6a9a93ab7737c118ae3b8509dc", size = 412623 }, + { url = "https://files.pythonhosted.org/packages/23/d5/7cbfbebbb3ffccb0626fc2fab622fb5a10cf66c2c60481f51e46a92eb2c5/pyzstd-0.16.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1c6c0a677aac7c0e3d2d2605d4d68ffa9893fdeeb2e071040eb7c8750969d463", size = 404981 }, + { url = "https://files.pythonhosted.org/packages/a7/b0/6ac198c753cc135357630e856f40f5998c2d28609713ae2830c679e8248c/pyzstd-0.16.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:15f9c2d612e7e2023d68d321d1b479846751f792af89141931d44e82ae391394", size = 417997 }, + { url = "https://files.pythonhosted.org/packages/c6/8f/0e5685efbf24ae62e135549e37947ca7919616b81108584112e25dd1a55a/pyzstd-0.16.2-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:11740bff847aad23beef4085a1bb767d101895881fe891f0a911aa27d43c372c", size = 485576 }, + { url = "https://files.pythonhosted.org/packages/30/d6/bf2f05752082967ac748d7c2d7c5a71097ac6fc1b902b5d34764cd0c12f7/pyzstd-0.16.2-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:b9067483ebe860e4130a03ee665b3d7be4ec1608b208e645d5e7eb3492379464", size = 564538 }, + { url = "https://files.pythonhosted.org/packages/d8/97/1081cc3cbf5eeb6cf4e385226e9989fdebb61f8e48baa210eb774145e667/pyzstd-0.16.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:988f0ba19b14c2fe0afefc444ac1edfb2f497b7d7c3212b2f587504cc2ec804e", size = 430615 }, + { url = "https://files.pythonhosted.org/packages/e0/a7/2a82fbb248b951434306dd77e969fb99305968904c9a7494574d696b1392/pyzstd-0.16.2-cp39-cp39-win32.whl", hash = "sha256:8855acb1c3e3829030b9e9e9973b19e2d70f33efb14ad5c474b4d086864c959c", size = 218215 }, + { url = "https://files.pythonhosted.org/packages/9d/bf/e529ff84b87c8f978ab35906921ac54841270562e65bcb5d0dd9d3240204/pyzstd-0.16.2-cp39-cp39-win_amd64.whl", hash = "sha256:018e88378df5e76f5e1d8cf4416576603b6bc4a103cbc66bb593eaac54c758de", size = 245047 }, + { url = "https://files.pythonhosted.org/packages/f9/ad/c09fb722c12a82b826c97efc50a919e229bfbaf644f5a140adcd71941473/pyzstd-0.16.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4b631117b97a42ff6dfd0ffc885a92fff462d7c34766b28383c57b996f863338", size = 364187 }, + { url = "https://files.pythonhosted.org/packages/57/f9/93175fe72f85fb675fe04abca296fe583112a25d0ec7faa026288d9463c2/pyzstd-0.16.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:56493a3fbe1b651a02102dd0902b0aa2377a732ff3544fb6fb3f114ca18db52f", size = 279825 }, + { url = "https://files.pythonhosted.org/packages/8a/de/0b40acf76d7ed1f7975877535e004de85ec2e869632754b5d4d389258b8a/pyzstd-0.16.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1eae9bdba4a1e5d3181331f403114ff5b8ce0f4b569f48eba2b9beb2deef1e4", size = 321313 }, + { url = "https://files.pythonhosted.org/packages/41/5e/00102bacd1a7c957c88098f3ae2cdac17842ac0f94d2e685ff5b75a05730/pyzstd-0.16.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1be6972391c8aeecc7e61feb96ffc8e77a401bcba6ed994e7171330c45a1948", size = 344376 }, + { url = "https://files.pythonhosted.org/packages/a3/95/27a7da3dbd4460cd9432bdc22d9d5f8ec77c86275d069020fa74ea280f7f/pyzstd-0.16.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:761439d687e3a5687c2ff5c6a1190e1601362a4a3e8c6c82ff89719d51d73e19", size = 328591 }, + { url = "https://files.pythonhosted.org/packages/c2/03/8f4d5fd45f6bfad66d67cdf583492a9f52a21049f60e6b36a7e9f8aa7adc/pyzstd-0.16.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f5fbdb8cf31b60b2dc586fecb9b73e2f172c21a0b320ed275f7b8d8a866d9003", size = 240786 }, + { url = "https://files.pythonhosted.org/packages/91/f6/bd63e2587e0ec40abd9f92278a442bc28b7ff109e418d1240ee2eb6536aa/pyzstd-0.16.2-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:183f26e34f9becf0f2db38be9c0bfb136753d228bcb47c06c69175901bea7776", size = 364180 }, + { url = "https://files.pythonhosted.org/packages/ac/13/d4c68ad926e79d734f57b26d49447908e8dab7f5c066d3a013b0d0cfa2be/pyzstd-0.16.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:88318b64b5205a67748148d6d244097fa6cf61fcea02ad3435511b9e7155ae16", size = 279816 }, + { url = "https://files.pythonhosted.org/packages/b2/ba/76f0b75ec9e9fc3914496e036f99f345d5e0a99cb7070341f9becdaba2b8/pyzstd-0.16.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73142aa2571b6480136a1865ebda8257e09eabbc8bcd54b222202f6fa4febe1e", size = 321308 }, + { url = "https://files.pythonhosted.org/packages/a6/ea/9fe52bd777f33f007287f1a37bada7af5cf33d64904360c17bb64fefca21/pyzstd-0.16.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d3f8877c29a97f1b1bba16f3d3ab01ad10ad3da7bad317aecf36aaf8848b37c", size = 344368 }, + { url = "https://files.pythonhosted.org/packages/cc/c0/509077f73fc8e156ceeefb41d4b7e04aceb71b2339084fcd62d0ad3bfd75/pyzstd-0.16.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1f25754562473ac7de856b8331ebd5964f5d85601045627a5f0bb0e4e899990", size = 328585 }, + { url = "https://files.pythonhosted.org/packages/14/74/a854ada61bf4c3c2ad239ec2bd1ff73cc0d718ccbcc56e3ced94e878fd50/pyzstd-0.16.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:6ce17e84310080c55c02827ad9bb17893c00a845c8386a328b346f814aabd2c1", size = 240783 }, +] + [[package]] name = "requests" version = "2.32.3" @@ -1682,26 +2095,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.1.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/25/4c/2f786388acd82c295eedc4afeede7ef4b29cf27277151d8d13be906bac70/ruff-0.1.6.tar.gz", hash = "sha256:1b09f29b16c6ead5ea6b097ef2764b42372aebe363722f1605ecbcd2b9207184", size = 1719627 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/75/5054ec93ec0d5db26e218cb2814ddaa085ba1f29fad0ec56dd8107a97688/ruff-0.1.6-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:88b8cdf6abf98130991cbc9f6438f35f6e8d41a02622cc5ee130a02a0ed28703", size = 11628468 }, - { url = "https://files.pythonhosted.org/packages/a2/91/8b2920f6026c069ae0802fc3c44f7337e04bf2a198ce94bfab360073477a/ruff-0.1.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5c549ed437680b6105a1299d2cd30e4964211606eeb48a0ff7a93ef70b902248", size = 5990066 }, - { url = "https://files.pythonhosted.org/packages/df/1e/03ef0cc5c7d03e50d4f954218551d6001f1f70e6f391cdb678efb5c6e6ab/ruff-0.1.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cf5f701062e294f2167e66d11b092bba7af6a057668ed618a9253e1e90cfd76", size = 5929313 }, - { url = "https://files.pythonhosted.org/packages/92/7c/38fd1b9cb624f5725a6a08c81bf7e823c64b28622ffcb4369c56dc0a16d0/ruff-0.1.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:05991ee20d4ac4bb78385360c684e4b417edd971030ab12a4fbd075ff535050e", size = 5627943 }, - { url = "https://files.pythonhosted.org/packages/c7/c3/98e3d0eb92e5a2ec10f76c71067640b6f21def23c3b1ff8f08ab6348255e/ruff-0.1.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87455a0c1f739b3c069e2f4c43b66479a54dea0276dd5d4d67b091265f6fd1dc", size = 6074206 }, - { url = "https://files.pythonhosted.org/packages/e8/33/62fb966eb70d9bb45ddf5023d40e26946a5e5127d99956b84c8a9a76b153/ruff-0.1.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:683aa5bdda5a48cb8266fcde8eea2a6af4e5700a392c56ea5fb5f0d4bfdc0240", size = 6743205 }, - { url = "https://files.pythonhosted.org/packages/c7/f1/60d43182f98113156a1b21a17f30541dda9f5ffcfeedc2b54dc030a2c413/ruff-0.1.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:137852105586dcbf80c1717facb6781555c4e99f520c9c827bd414fac67ddfb6", size = 6600581 }, - { url = "https://files.pythonhosted.org/packages/09/92/36850598e84f75cfe8edd252dbf40442b4cc226ed2c76206a9b3cbfb9986/ruff-0.1.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd98138a98d48a1c36c394fd6b84cd943ac92a08278aa8ac8c0fdefcf7138f35", size = 7708098 }, - { url = "https://files.pythonhosted.org/packages/3b/2f/8ef67614631622aa3ea79b27e01ac86d7f90a988520454e3a84cb2fd890f/ruff-0.1.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a0cd909d25f227ac5c36d4e7e681577275fb74ba3b11d288aff7ec47e3ae745", size = 6295145 }, - { url = "https://files.pythonhosted.org/packages/3c/4b/af366db98d15efe83fd3e3aae7319d3897e3475fc53a2f1b0287c8255422/ruff-0.1.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e8fd1c62a47aa88a02707b5dd20c5ff20d035d634aa74826b42a1da77861b5ff", size = 5878111 }, - { url = "https://files.pythonhosted.org/packages/bf/af/25b794e750f1d74a83ce6b16625e3306beeb2161c517b9d883958de05526/ruff-0.1.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fd89b45d374935829134a082617954120d7a1470a9f0ec0e7f3ead983edc48cc", size = 5629339 }, - { url = "https://files.pythonhosted.org/packages/81/b0/92c4cb6bceb19ebd27cedd1f45b337f7fd5397e6b760094831266be59661/ruff-0.1.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:491262006e92f825b145cd1e52948073c56560243b55fb3b4ecb142f6f0e9543", size = 5959772 }, - { url = "https://files.pythonhosted.org/packages/11/02/3a7e3101d88b113f326e0fdf3f566fba2600fc4b1fd828d56027d293e22d/ruff-0.1.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ea284789861b8b5ca9d5443591a92a397ac183d4351882ab52f6296b4fdd5462", size = 6356759 }, - { url = "https://files.pythonhosted.org/packages/fc/93/8a4b3523c4564168aab720d8361b4bf12173b099509caaac93168b72927a/ruff-0.1.6-py3-none-win32.whl", hash = "sha256:1610e14750826dfc207ccbcdd7331b6bd285607d4181df9c1c6ae26646d6848a", size = 5835632 }, - { url = "https://files.pythonhosted.org/packages/40/e3/55ae013087bd892dd61c25895a119ffec2f9b9f7c3d1fd0cea5f5f7bd74a/ruff-0.1.6-py3-none-win_amd64.whl", hash = "sha256:4558b3e178145491e9bc3b2ee3c4b42f19d19384eaa5c59d10acf6e8f8b57e33", size = 6226075 }, - { url = "https://files.pythonhosted.org/packages/21/f7/fdce733e594e1b274fc9232256be6a4e03e8d2cb9e354c783801191e2f4d/ruff-0.1.6-py3-none-win_arm64.whl", hash = "sha256:03910e81df0d8db0e30050725a5802441c2022ea3ae4fe0609b76081731accbc", size = 5965823 }, +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/2b/01245f4f3a727d60bebeacd7ee6d22586c7f62380a2597ddb22c2f45d018/ruff-0.8.2.tar.gz", hash = "sha256:b84f4f414dda8ac7f75075c1fa0b905ac0ff25361f42e6d5da681a465e0f78e5", size = 3349020 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/29/366be70216dba1731a00a41f2f030822b0c96c7c4f3b2c0cdce15cbace74/ruff-0.8.2-py3-none-linux_armv6l.whl", hash = "sha256:c49ab4da37e7c457105aadfd2725e24305ff9bc908487a9bf8d548c6dad8bb3d", size = 10530649 }, + { url = "https://files.pythonhosted.org/packages/63/82/a733956540bb388f00df5a3e6a02467b16c0e529132625fe44ce4c5fb9c7/ruff-0.8.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ec016beb69ac16be416c435828be702ee694c0d722505f9c1f35e1b9c0cc1bf5", size = 10274069 }, + { url = "https://files.pythonhosted.org/packages/3d/12/0b3aa14d1d71546c988a28e1b412981c1b80c8a1072e977a2f30c595cc4a/ruff-0.8.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:f05cdf8d050b30e2ba55c9b09330b51f9f97d36d4673213679b965d25a785f3c", size = 9909400 }, + { url = "https://files.pythonhosted.org/packages/23/08/f9f08cefb7921784c891c4151cce6ed357ff49e84b84978440cffbc87408/ruff-0.8.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:60f578c11feb1d3d257b2fb043ddb47501ab4816e7e221fbb0077f0d5d4e7b6f", size = 10766782 }, + { url = "https://files.pythonhosted.org/packages/e4/71/bf50c321ec179aa420c8ec40adac5ae9cc408d4d37283a485b19a2331ceb/ruff-0.8.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cbd5cf9b0ae8f30eebc7b360171bd50f59ab29d39f06a670b3e4501a36ba5897", size = 10286316 }, + { url = "https://files.pythonhosted.org/packages/f2/83/c82688a2a6117539aea0ce63fdf6c08e60fe0202779361223bcd7f40bd74/ruff-0.8.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b402ddee3d777683de60ff76da801fa7e5e8a71038f57ee53e903afbcefdaa58", size = 11338270 }, + { url = "https://files.pythonhosted.org/packages/7f/d7/bc6a45e5a22e627640388e703160afb1d77c572b1d0fda8b4349f334fc66/ruff-0.8.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:705832cd7d85605cb7858d8a13d75993c8f3ef1397b0831289109e953d833d29", size = 12058579 }, + { url = "https://files.pythonhosted.org/packages/da/3b/64150c93946ec851e6f1707ff586bb460ca671581380c919698d6a9267dc/ruff-0.8.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:32096b41aaf7a5cc095fa45b4167b890e4c8d3fd217603f3634c92a541de7248", size = 11615172 }, + { url = "https://files.pythonhosted.org/packages/e4/9e/cf12b697ea83cfe92ec4509ae414dc4c9b38179cc681a497031f0d0d9a8e/ruff-0.8.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e769083da9439508833cfc7c23e351e1809e67f47c50248250ce1ac52c21fb93", size = 12882398 }, + { url = "https://files.pythonhosted.org/packages/a9/27/96d10863accf76a9c97baceac30b0a52d917eb985a8ac058bd4636aeede0/ruff-0.8.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fe716592ae8a376c2673fdfc1f5c0c193a6d0411f90a496863c99cd9e2ae25d", size = 11176094 }, + { url = "https://files.pythonhosted.org/packages/eb/10/cd2fd77d4a4e7f03c29351be0f53278a393186b540b99df68beb5304fddd/ruff-0.8.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:81c148825277e737493242b44c5388a300584d73d5774defa9245aaef55448b0", size = 10771884 }, + { url = "https://files.pythonhosted.org/packages/71/5d/beabb2ff18870fc4add05fa3a69a4cb1b1d2d6f83f3cf3ae5ab0d52f455d/ruff-0.8.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d261d7850c8367704874847d95febc698a950bf061c9475d4a8b7689adc4f7fa", size = 10382535 }, + { url = "https://files.pythonhosted.org/packages/ae/29/6b3fdf3ad3e35b28d87c25a9ff4c8222ad72485ab783936b2b267250d7a7/ruff-0.8.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1ca4e3a87496dc07d2427b7dd7ffa88a1e597c28dad65ae6433ecb9f2e4f022f", size = 10886995 }, + { url = "https://files.pythonhosted.org/packages/e9/dc/859d889b4d9356a1a2cdbc1e4a0dda94052bc5b5300098647e51a58c430b/ruff-0.8.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:729850feed82ef2440aa27946ab39c18cb4a8889c1128a6d589ffa028ddcfc22", size = 11220750 }, + { url = "https://files.pythonhosted.org/packages/0b/08/e8f519f61f1d624264bfd6b8829e4c5f31c3c61193bc3cff1f19dbe7626a/ruff-0.8.2-py3-none-win32.whl", hash = "sha256:ac42caaa0411d6a7d9594363294416e0e48fc1279e1b0e948391695db2b3d5b1", size = 8729396 }, + { url = "https://files.pythonhosted.org/packages/f8/d4/ba1c7ab72aba37a2b71fe48ab95b80546dbad7a7f35ea28cf66fc5cea5f6/ruff-0.8.2-py3-none-win_amd64.whl", hash = "sha256:2aae99ec70abf43372612a838d97bfe77d45146254568d94926e8ed5bbb409ea", size = 9594729 }, + { url = "https://files.pythonhosted.org/packages/23/34/db20e12d3db11b8a2a8874258f0f6d96a9a4d631659d54575840557164c8/ruff-0.8.2-py3-none-win_arm64.whl", hash = "sha256:fb88e2a506b70cfbc2de6fae6681c4f944f7dd5f2fe87233a7233d888bad73e8", size = 9035131 }, ] [[package]] @@ -1860,6 +2274,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bf/65/813fc133609ebcb1299be6a42e5aea99d6344afb35ccb43f67e7daaa3b92/structlog-24.4.0-py3-none-any.whl", hash = "sha256:597f61e80a91cc0749a9fd2a098ed76715a1c8a01f73e336b746504d1aad7610", size = 67180 }, ] +[[package]] +name = "texttable" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/dc/0aff23d6036a4d3bf4f1d8c8204c5c79c4437e25e0ae94ffe4bbb55ee3c2/texttable-1.7.0.tar.gz", hash = "sha256:2d2068fb55115807d3ac77a4ca68fa48803e84ebb0ee2340f858107a36522638", size = 12831 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl", hash = "sha256:72227d592c82b3d7f672731ae73e4d1f88cd8e2ef5b075a7a7f01a23a3743917", size = 10768 }, +] + [[package]] name = "tomli" version = "2.0.1" diff --git a/lib/repack/owid/repack/__init__.py b/lib/repack/owid/repack/__init__.py index 6dbc604bc64..4772c91c012 100644 --- a/lib/repack/owid/repack/__init__.py +++ b/lib/repack/owid/repack/__init__.py @@ -77,9 +77,10 @@ def repack_series(s: pd.Series) -> pd.Series: def _to_float(s: pd.Series) -> pd.Series: """Convert series to Float64. Replace numpy NaNs with NA. This can happen when original series is an object and contains 'nan' string.""" - s = s.astype("Float64") - s = s.mask(np.isnan(s), pd.NA) - return s + r = s.astype("Float64") + if s.dtype == "object": + r = r.mask(np.isnan(r), pd.NA) + return r def to_int(s: pd.Series) -> pd.Series: @@ -173,7 +174,7 @@ def _safe_dtype(dtype: Any) -> str: return "boolean" elif isinstance(dtype, pd.CategoricalDtype): return "string[pyarrow]" - elif dtype == object: + elif dtype == "object": return "string[pyarrow]" else: return dtype diff --git a/lib/repack/pyproject.toml b/lib/repack/pyproject.toml index fea1f71e18c..1c54e179ebe 100644 --- a/lib/repack/pyproject.toml +++ b/lib/repack/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ dev-dependencies = [ "pytest>=7.2.0", "pyright==1.1.373", - "ruff==0.1.6", + "ruff==0.8.2", "ipdb>=0.13.13", ] diff --git a/lib/repack/uv.lock b/lib/repack/uv.lock index 1e54dea67bf..a8132080e4e 100644 --- a/lib/repack/uv.lock +++ b/lib/repack/uv.lock @@ -205,7 +205,7 @@ dev = [ { name = "ipdb", specifier = ">=0.13.13" }, { name = "pyright", specifier = "==1.1.373" }, { name = "pytest", specifier = ">=7.2.0" }, - { name = "ruff", specifier = "==0.1.6" }, + { name = "ruff", specifier = "==0.8.2" }, ] [[package]] @@ -432,26 +432,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.1.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/25/4c/2f786388acd82c295eedc4afeede7ef4b29cf27277151d8d13be906bac70/ruff-0.1.6.tar.gz", hash = "sha256:1b09f29b16c6ead5ea6b097ef2764b42372aebe363722f1605ecbcd2b9207184", size = 1719627 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/75/5054ec93ec0d5db26e218cb2814ddaa085ba1f29fad0ec56dd8107a97688/ruff-0.1.6-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:88b8cdf6abf98130991cbc9f6438f35f6e8d41a02622cc5ee130a02a0ed28703", size = 11628468 }, - { url = "https://files.pythonhosted.org/packages/a2/91/8b2920f6026c069ae0802fc3c44f7337e04bf2a198ce94bfab360073477a/ruff-0.1.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5c549ed437680b6105a1299d2cd30e4964211606eeb48a0ff7a93ef70b902248", size = 5990066 }, - { url = "https://files.pythonhosted.org/packages/df/1e/03ef0cc5c7d03e50d4f954218551d6001f1f70e6f391cdb678efb5c6e6ab/ruff-0.1.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cf5f701062e294f2167e66d11b092bba7af6a057668ed618a9253e1e90cfd76", size = 5929313 }, - { url = "https://files.pythonhosted.org/packages/92/7c/38fd1b9cb624f5725a6a08c81bf7e823c64b28622ffcb4369c56dc0a16d0/ruff-0.1.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:05991ee20d4ac4bb78385360c684e4b417edd971030ab12a4fbd075ff535050e", size = 5627943 }, - { url = "https://files.pythonhosted.org/packages/c7/c3/98e3d0eb92e5a2ec10f76c71067640b6f21def23c3b1ff8f08ab6348255e/ruff-0.1.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87455a0c1f739b3c069e2f4c43b66479a54dea0276dd5d4d67b091265f6fd1dc", size = 6074206 }, - { url = "https://files.pythonhosted.org/packages/e8/33/62fb966eb70d9bb45ddf5023d40e26946a5e5127d99956b84c8a9a76b153/ruff-0.1.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:683aa5bdda5a48cb8266fcde8eea2a6af4e5700a392c56ea5fb5f0d4bfdc0240", size = 6743205 }, - { url = "https://files.pythonhosted.org/packages/c7/f1/60d43182f98113156a1b21a17f30541dda9f5ffcfeedc2b54dc030a2c413/ruff-0.1.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:137852105586dcbf80c1717facb6781555c4e99f520c9c827bd414fac67ddfb6", size = 6600581 }, - { url = "https://files.pythonhosted.org/packages/09/92/36850598e84f75cfe8edd252dbf40442b4cc226ed2c76206a9b3cbfb9986/ruff-0.1.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd98138a98d48a1c36c394fd6b84cd943ac92a08278aa8ac8c0fdefcf7138f35", size = 7708098 }, - { url = "https://files.pythonhosted.org/packages/3b/2f/8ef67614631622aa3ea79b27e01ac86d7f90a988520454e3a84cb2fd890f/ruff-0.1.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a0cd909d25f227ac5c36d4e7e681577275fb74ba3b11d288aff7ec47e3ae745", size = 6295145 }, - { url = "https://files.pythonhosted.org/packages/3c/4b/af366db98d15efe83fd3e3aae7319d3897e3475fc53a2f1b0287c8255422/ruff-0.1.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e8fd1c62a47aa88a02707b5dd20c5ff20d035d634aa74826b42a1da77861b5ff", size = 5878111 }, - { url = "https://files.pythonhosted.org/packages/bf/af/25b794e750f1d74a83ce6b16625e3306beeb2161c517b9d883958de05526/ruff-0.1.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fd89b45d374935829134a082617954120d7a1470a9f0ec0e7f3ead983edc48cc", size = 5629339 }, - { url = "https://files.pythonhosted.org/packages/81/b0/92c4cb6bceb19ebd27cedd1f45b337f7fd5397e6b760094831266be59661/ruff-0.1.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:491262006e92f825b145cd1e52948073c56560243b55fb3b4ecb142f6f0e9543", size = 5959772 }, - { url = "https://files.pythonhosted.org/packages/11/02/3a7e3101d88b113f326e0fdf3f566fba2600fc4b1fd828d56027d293e22d/ruff-0.1.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ea284789861b8b5ca9d5443591a92a397ac183d4351882ab52f6296b4fdd5462", size = 6356759 }, - { url = "https://files.pythonhosted.org/packages/fc/93/8a4b3523c4564168aab720d8361b4bf12173b099509caaac93168b72927a/ruff-0.1.6-py3-none-win32.whl", hash = "sha256:1610e14750826dfc207ccbcdd7331b6bd285607d4181df9c1c6ae26646d6848a", size = 5835632 }, - { url = "https://files.pythonhosted.org/packages/40/e3/55ae013087bd892dd61c25895a119ffec2f9b9f7c3d1fd0cea5f5f7bd74a/ruff-0.1.6-py3-none-win_amd64.whl", hash = "sha256:4558b3e178145491e9bc3b2ee3c4b42f19d19384eaa5c59d10acf6e8f8b57e33", size = 6226075 }, - { url = "https://files.pythonhosted.org/packages/21/f7/fdce733e594e1b274fc9232256be6a4e03e8d2cb9e354c783801191e2f4d/ruff-0.1.6-py3-none-win_arm64.whl", hash = "sha256:03910e81df0d8db0e30050725a5802441c2022ea3ae4fe0609b76081731accbc", size = 5965823 }, +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/2b/01245f4f3a727d60bebeacd7ee6d22586c7f62380a2597ddb22c2f45d018/ruff-0.8.2.tar.gz", hash = "sha256:b84f4f414dda8ac7f75075c1fa0b905ac0ff25361f42e6d5da681a465e0f78e5", size = 3349020 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/29/366be70216dba1731a00a41f2f030822b0c96c7c4f3b2c0cdce15cbace74/ruff-0.8.2-py3-none-linux_armv6l.whl", hash = "sha256:c49ab4da37e7c457105aadfd2725e24305ff9bc908487a9bf8d548c6dad8bb3d", size = 10530649 }, + { url = "https://files.pythonhosted.org/packages/63/82/a733956540bb388f00df5a3e6a02467b16c0e529132625fe44ce4c5fb9c7/ruff-0.8.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ec016beb69ac16be416c435828be702ee694c0d722505f9c1f35e1b9c0cc1bf5", size = 10274069 }, + { url = "https://files.pythonhosted.org/packages/3d/12/0b3aa14d1d71546c988a28e1b412981c1b80c8a1072e977a2f30c595cc4a/ruff-0.8.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:f05cdf8d050b30e2ba55c9b09330b51f9f97d36d4673213679b965d25a785f3c", size = 9909400 }, + { url = "https://files.pythonhosted.org/packages/23/08/f9f08cefb7921784c891c4151cce6ed357ff49e84b84978440cffbc87408/ruff-0.8.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:60f578c11feb1d3d257b2fb043ddb47501ab4816e7e221fbb0077f0d5d4e7b6f", size = 10766782 }, + { url = "https://files.pythonhosted.org/packages/e4/71/bf50c321ec179aa420c8ec40adac5ae9cc408d4d37283a485b19a2331ceb/ruff-0.8.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cbd5cf9b0ae8f30eebc7b360171bd50f59ab29d39f06a670b3e4501a36ba5897", size = 10286316 }, + { url = "https://files.pythonhosted.org/packages/f2/83/c82688a2a6117539aea0ce63fdf6c08e60fe0202779361223bcd7f40bd74/ruff-0.8.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b402ddee3d777683de60ff76da801fa7e5e8a71038f57ee53e903afbcefdaa58", size = 11338270 }, + { url = "https://files.pythonhosted.org/packages/7f/d7/bc6a45e5a22e627640388e703160afb1d77c572b1d0fda8b4349f334fc66/ruff-0.8.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:705832cd7d85605cb7858d8a13d75993c8f3ef1397b0831289109e953d833d29", size = 12058579 }, + { url = "https://files.pythonhosted.org/packages/da/3b/64150c93946ec851e6f1707ff586bb460ca671581380c919698d6a9267dc/ruff-0.8.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:32096b41aaf7a5cc095fa45b4167b890e4c8d3fd217603f3634c92a541de7248", size = 11615172 }, + { url = "https://files.pythonhosted.org/packages/e4/9e/cf12b697ea83cfe92ec4509ae414dc4c9b38179cc681a497031f0d0d9a8e/ruff-0.8.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e769083da9439508833cfc7c23e351e1809e67f47c50248250ce1ac52c21fb93", size = 12882398 }, + { url = "https://files.pythonhosted.org/packages/a9/27/96d10863accf76a9c97baceac30b0a52d917eb985a8ac058bd4636aeede0/ruff-0.8.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fe716592ae8a376c2673fdfc1f5c0c193a6d0411f90a496863c99cd9e2ae25d", size = 11176094 }, + { url = "https://files.pythonhosted.org/packages/eb/10/cd2fd77d4a4e7f03c29351be0f53278a393186b540b99df68beb5304fddd/ruff-0.8.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:81c148825277e737493242b44c5388a300584d73d5774defa9245aaef55448b0", size = 10771884 }, + { url = "https://files.pythonhosted.org/packages/71/5d/beabb2ff18870fc4add05fa3a69a4cb1b1d2d6f83f3cf3ae5ab0d52f455d/ruff-0.8.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d261d7850c8367704874847d95febc698a950bf061c9475d4a8b7689adc4f7fa", size = 10382535 }, + { url = "https://files.pythonhosted.org/packages/ae/29/6b3fdf3ad3e35b28d87c25a9ff4c8222ad72485ab783936b2b267250d7a7/ruff-0.8.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1ca4e3a87496dc07d2427b7dd7ffa88a1e597c28dad65ae6433ecb9f2e4f022f", size = 10886995 }, + { url = "https://files.pythonhosted.org/packages/e9/dc/859d889b4d9356a1a2cdbc1e4a0dda94052bc5b5300098647e51a58c430b/ruff-0.8.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:729850feed82ef2440aa27946ab39c18cb4a8889c1128a6d589ffa028ddcfc22", size = 11220750 }, + { url = "https://files.pythonhosted.org/packages/0b/08/e8f519f61f1d624264bfd6b8829e4c5f31c3c61193bc3cff1f19dbe7626a/ruff-0.8.2-py3-none-win32.whl", hash = "sha256:ac42caaa0411d6a7d9594363294416e0e48fc1279e1b0e948391695db2b3d5b1", size = 8729396 }, + { url = "https://files.pythonhosted.org/packages/f8/d4/ba1c7ab72aba37a2b71fe48ab95b80546dbad7a7f35ea28cf66fc5cea5f6/ruff-0.8.2-py3-none-win_amd64.whl", hash = "sha256:2aae99ec70abf43372612a838d97bfe77d45146254568d94926e8ed5bbb409ea", size = 9594729 }, + { url = "https://files.pythonhosted.org/packages/23/34/db20e12d3db11b8a2a8874258f0f6d96a9a4d631659d54575840557164c8/ruff-0.8.2-py3-none-win_arm64.whl", hash = "sha256:fb88e2a506b70cfbc2de6fae6681c4f944f7dd5f2fe87233a7233d888bad73e8", size = 9035131 }, ] [[package]] diff --git a/lib/walden/owid/walden/catalog.py b/lib/walden/owid/walden/catalog.py index d7c53f3972e..b5d4ef390b2 100644 --- a/lib/walden/owid/walden/catalog.py +++ b/lib/walden/owid/walden/catalog.py @@ -272,8 +272,7 @@ def delete_from_remote(self) -> None: def local_path(self) -> str: return path.join(CACHE_DIR, f"{self.relative_base}.{self.file_extension}") - def to_dict(self) -> Dict[str, Any]: - ... # type: ignore + def to_dict(self) -> Dict[str, Any]: ... # type: ignore def has_changed_from_last_version(self) -> bool: """Check if local dataset is different to latest available version in Walden. diff --git a/lib/walden/pyproject.toml b/lib/walden/pyproject.toml index 954adc93985..1016cea4a48 100644 --- a/lib/walden/pyproject.toml +++ b/lib/walden/pyproject.toml @@ -37,7 +37,7 @@ dev-dependencies = [ "jupyter_nbextensions_configurator>=0.4.1", "types-PyYAML>=6.0.5", "pyright==1.1.373", - "ruff==0.1.6", + "ruff==0.8.2", ] [tool.uv.sources] diff --git a/lib/walden/uv.lock b/lib/walden/uv.lock index 3a8d2678812..83bace54115 100644 --- a/lib/walden/uv.lock +++ b/lib/walden/uv.lock @@ -181,6 +181,119 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/2e/ca478472e3a6cc96a23dcaf82af714e2befbf449aec98974bf0ac2c88102/botocore-1.35.20-py3-none-any.whl", hash = "sha256:62412038f960691a299e60492f9ee7e8e75af563f2eca7f3640b3b54b8f5d236", size = 12539792 }, ] +[[package]] +name = "brotli" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/c2/f9e977608bdf958650638c3f1e28f85a1b075f075ebbe77db8555463787b/Brotli-1.1.0.tar.gz", hash = "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724", size = 7372270 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/3a/dbf4fb970c1019a57b5e492e1e0eae745d32e59ba4d6161ab5422b08eefe/Brotli-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e1140c64812cb9b06c922e77f1c26a75ec5e3f0fb2bf92cc8c58720dec276752", size = 873045 }, + { url = "https://files.pythonhosted.org/packages/dd/11/afc14026ea7f44bd6eb9316d800d439d092c8d508752055ce8d03086079a/Brotli-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c8fd5270e906eef71d4a8d19b7c6a43760c6abcfcc10c9101d14eb2357418de9", size = 446218 }, + { url = "https://files.pythonhosted.org/packages/36/83/7545a6e7729db43cb36c4287ae388d6885c85a86dd251768a47015dfde32/Brotli-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ae56aca0402a0f9a3431cddda62ad71666ca9d4dc3a10a142b9dce2e3c0cda3", size = 2903872 }, + { url = "https://files.pythonhosted.org/packages/32/23/35331c4d9391fcc0f29fd9bec2c76e4b4eeab769afbc4b11dd2e1098fb13/Brotli-1.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:43ce1b9935bfa1ede40028054d7f48b5469cd02733a365eec8a329ffd342915d", size = 2941254 }, + { url = "https://files.pythonhosted.org/packages/3b/24/1671acb450c902edb64bd765d73603797c6c7280a9ada85a195f6b78c6e5/Brotli-1.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:7c4855522edb2e6ae7fdb58e07c3ba9111e7621a8956f481c68d5d979c93032e", size = 2857293 }, + { url = "https://files.pythonhosted.org/packages/d5/00/40f760cc27007912b327fe15bf6bfd8eaecbe451687f72a8abc587d503b3/Brotli-1.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:38025d9f30cf4634f8309c6874ef871b841eb3c347e90b0851f63d1ded5212da", size = 3002385 }, + { url = "https://files.pythonhosted.org/packages/b8/cb/8aaa83f7a4caa131757668c0fb0c4b6384b09ffa77f2fba9570d87ab587d/Brotli-1.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e6a904cb26bfefc2f0a6f240bdf5233be78cd2488900a2f846f3c3ac8489ab80", size = 2911104 }, + { url = "https://files.pythonhosted.org/packages/bc/c4/65456561d89d3c49f46b7fbeb8fe6e449f13bdc8ea7791832c5d476b2faf/Brotli-1.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a37b8f0391212d29b3a91a799c8e4a2855e0576911cdfb2515487e30e322253d", size = 2809981 }, + { url = "https://files.pythonhosted.org/packages/05/1b/cf49528437bae28abce5f6e059f0d0be6fecdcc1d3e33e7c54b3ca498425/Brotli-1.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e84799f09591700a4154154cab9787452925578841a94321d5ee8fb9a9a328f0", size = 2935297 }, + { url = "https://files.pythonhosted.org/packages/81/ff/190d4af610680bf0c5a09eb5d1eac6e99c7c8e216440f9c7cfd42b7adab5/Brotli-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f66b5337fa213f1da0d9000bc8dc0cb5b896b726eefd9c6046f699b169c41b9e", size = 2930735 }, + { url = "https://files.pythonhosted.org/packages/80/7d/f1abbc0c98f6e09abd3cad63ec34af17abc4c44f308a7a539010f79aae7a/Brotli-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5dab0844f2cf82be357a0eb11a9087f70c5430b2c241493fc122bb6f2bb0917c", size = 2933107 }, + { url = "https://files.pythonhosted.org/packages/34/ce/5a5020ba48f2b5a4ad1c0522d095ad5847a0be508e7d7569c8630ce25062/Brotli-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e4fe605b917c70283db7dfe5ada75e04561479075761a0b3866c081d035b01c1", size = 2845400 }, + { url = "https://files.pythonhosted.org/packages/44/89/fa2c4355ab1eecf3994e5a0a7f5492c6ff81dfcb5f9ba7859bd534bb5c1a/Brotli-1.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1e9a65b5736232e7a7f91ff3d02277f11d339bf34099a56cdab6a8b3410a02b2", size = 3031985 }, + { url = "https://files.pythonhosted.org/packages/af/a4/79196b4a1674143d19dca400866b1a4d1a089040df7b93b88ebae81f3447/Brotli-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:58d4b711689366d4a03ac7957ab8c28890415e267f9b6589969e74b6e42225ec", size = 2927099 }, + { url = "https://files.pythonhosted.org/packages/e9/54/1c0278556a097f9651e657b873ab08f01b9a9ae4cac128ceb66427d7cd20/Brotli-1.1.0-cp310-cp310-win32.whl", hash = "sha256:be36e3d172dc816333f33520154d708a2657ea63762ec16b62ece02ab5e4daf2", size = 333172 }, + { url = "https://files.pythonhosted.org/packages/f7/65/b785722e941193fd8b571afd9edbec2a9b838ddec4375d8af33a50b8dab9/Brotli-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:0c6244521dda65ea562d5a69b9a26120769b7a9fb3db2fe9545935ed6735b128", size = 357255 }, + { url = "https://files.pythonhosted.org/packages/96/12/ad41e7fadd5db55459c4c401842b47f7fee51068f86dd2894dd0dcfc2d2a/Brotli-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a3daabb76a78f829cafc365531c972016e4aa8d5b4bf60660ad8ecee19df7ccc", size = 873068 }, + { url = "https://files.pythonhosted.org/packages/95/4e/5afab7b2b4b61a84e9c75b17814198ce515343a44e2ed4488fac314cd0a9/Brotli-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c8146669223164fc87a7e3de9f81e9423c67a79d6b3447994dfb9c95da16e2d6", size = 446244 }, + { url = "https://files.pythonhosted.org/packages/9d/e6/f305eb61fb9a8580c525478a4a34c5ae1a9bcb12c3aee619114940bc513d/Brotli-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30924eb4c57903d5a7526b08ef4a584acc22ab1ffa085faceb521521d2de32dd", size = 2906500 }, + { url = "https://files.pythonhosted.org/packages/3e/4f/af6846cfbc1550a3024e5d3775ede1e00474c40882c7bf5b37a43ca35e91/Brotli-1.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ceb64bbc6eac5a140ca649003756940f8d6a7c444a68af170b3187623b43bebf", size = 2943950 }, + { url = "https://files.pythonhosted.org/packages/b3/e7/ca2993c7682d8629b62630ebf0d1f3bb3d579e667ce8e7ca03a0a0576a2d/Brotli-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a469274ad18dc0e4d316eefa616d1d0c2ff9da369af19fa6f3daa4f09671fd61", size = 2918527 }, + { url = "https://files.pythonhosted.org/packages/b3/96/da98e7bedc4c51104d29cc61e5f449a502dd3dbc211944546a4cc65500d3/Brotli-1.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:524f35912131cc2cabb00edfd8d573b07f2d9f21fa824bd3fb19725a9cf06327", size = 2845489 }, + { url = "https://files.pythonhosted.org/packages/e8/ef/ccbc16947d6ce943a7f57e1a40596c75859eeb6d279c6994eddd69615265/Brotli-1.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5b3cc074004d968722f51e550b41a27be656ec48f8afaeeb45ebf65b561481dd", size = 2914080 }, + { url = "https://files.pythonhosted.org/packages/80/d6/0bd38d758d1afa62a5524172f0b18626bb2392d717ff94806f741fcd5ee9/Brotli-1.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:19c116e796420b0cee3da1ccec3b764ed2952ccfcc298b55a10e5610ad7885f9", size = 2813051 }, + { url = "https://files.pythonhosted.org/packages/14/56/48859dd5d129d7519e001f06dcfbb6e2cf6db92b2702c0c2ce7d97e086c1/Brotli-1.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:510b5b1bfbe20e1a7b3baf5fed9e9451873559a976c1a78eebaa3b86c57b4265", size = 2938172 }, + { url = "https://files.pythonhosted.org/packages/3d/77/a236d5f8cd9e9f4348da5acc75ab032ab1ab2c03cc8f430d24eea2672888/Brotli-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a1fd8a29719ccce974d523580987b7f8229aeace506952fa9ce1d53a033873c8", size = 2933023 }, + { url = "https://files.pythonhosted.org/packages/f1/87/3b283efc0f5cb35f7f84c0c240b1e1a1003a5e47141a4881bf87c86d0ce2/Brotli-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c247dd99d39e0338a604f8c2b3bc7061d5c2e9e2ac7ba9cc1be5a69cb6cd832f", size = 2935871 }, + { url = "https://files.pythonhosted.org/packages/f3/eb/2be4cc3e2141dc1a43ad4ca1875a72088229de38c68e842746b342667b2a/Brotli-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1b2c248cd517c222d89e74669a4adfa5577e06ab68771a529060cf5a156e9757", size = 2847784 }, + { url = "https://files.pythonhosted.org/packages/66/13/b58ddebfd35edde572ccefe6890cf7c493f0c319aad2a5badee134b4d8ec/Brotli-1.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:2a24c50840d89ded6c9a8fdc7b6ed3692ed4e86f1c4a4a938e1e92def92933e0", size = 3034905 }, + { url = "https://files.pythonhosted.org/packages/84/9c/bc96b6c7db824998a49ed3b38e441a2cae9234da6fa11f6ed17e8cf4f147/Brotli-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f31859074d57b4639318523d6ffdca586ace54271a73ad23ad021acd807eb14b", size = 2929467 }, + { url = "https://files.pythonhosted.org/packages/e7/71/8f161dee223c7ff7fea9d44893fba953ce97cf2c3c33f78ba260a91bcff5/Brotli-1.1.0-cp311-cp311-win32.whl", hash = "sha256:39da8adedf6942d76dc3e46653e52df937a3c4d6d18fdc94a7c29d263b1f5b50", size = 333169 }, + { url = "https://files.pythonhosted.org/packages/02/8a/fece0ee1057643cb2a5bbf59682de13f1725f8482b2c057d4e799d7ade75/Brotli-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:aac0411d20e345dc0920bdec5548e438e999ff68d77564d5e9463a7ca9d3e7b1", size = 357253 }, + { url = "https://files.pythonhosted.org/packages/5c/d0/5373ae13b93fe00095a58efcbce837fd470ca39f703a235d2a999baadfbc/Brotli-1.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:32d95b80260d79926f5fab3c41701dbb818fde1c9da590e77e571eefd14abe28", size = 815693 }, + { url = "https://files.pythonhosted.org/packages/8e/48/f6e1cdf86751300c288c1459724bfa6917a80e30dbfc326f92cea5d3683a/Brotli-1.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b760c65308ff1e462f65d69c12e4ae085cff3b332d894637f6273a12a482d09f", size = 422489 }, + { url = "https://files.pythonhosted.org/packages/06/88/564958cedce636d0f1bed313381dfc4b4e3d3f6015a63dae6146e1b8c65c/Brotli-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409", size = 873081 }, + { url = "https://files.pythonhosted.org/packages/58/79/b7026a8bb65da9a6bb7d14329fd2bd48d2b7f86d7329d5cc8ddc6a90526f/Brotli-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2", size = 446244 }, + { url = "https://files.pythonhosted.org/packages/e5/18/c18c32ecea41b6c0004e15606e274006366fe19436b6adccc1ae7b2e50c2/Brotli-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451", size = 2906505 }, + { url = "https://files.pythonhosted.org/packages/08/c8/69ec0496b1ada7569b62d85893d928e865df29b90736558d6c98c2031208/Brotli-1.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7f4bf76817c14aa98cc6697ac02f3972cb8c3da93e9ef16b9c66573a68014f91", size = 2944152 }, + { url = "https://files.pythonhosted.org/packages/ab/fb/0517cea182219d6768113a38167ef6d4eb157a033178cc938033a552ed6d/Brotli-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0c5516f0aed654134a2fc936325cc2e642f8a0e096d075209672eb321cff408", size = 2919252 }, + { url = "https://files.pythonhosted.org/packages/c7/53/73a3431662e33ae61a5c80b1b9d2d18f58dfa910ae8dd696e57d39f1a2f5/Brotli-1.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c3020404e0b5eefd7c9485ccf8393cfb75ec38ce75586e046573c9dc29967a0", size = 2845955 }, + { url = "https://files.pythonhosted.org/packages/55/ac/bd280708d9c5ebdbf9de01459e625a3e3803cce0784f47d633562cf40e83/Brotli-1.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4ed11165dd45ce798d99a136808a794a748d5dc38511303239d4e2363c0695dc", size = 2914304 }, + { url = "https://files.pythonhosted.org/packages/76/58/5c391b41ecfc4527d2cc3350719b02e87cb424ef8ba2023fb662f9bf743c/Brotli-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180", size = 2814452 }, + { url = "https://files.pythonhosted.org/packages/c7/4e/91b8256dfe99c407f174924b65a01f5305e303f486cc7a2e8a5d43c8bec3/Brotli-1.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248", size = 2938751 }, + { url = "https://files.pythonhosted.org/packages/5a/a6/e2a39a5d3b412938362bbbeba5af904092bf3f95b867b4a3eb856104074e/Brotli-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966", size = 2933757 }, + { url = "https://files.pythonhosted.org/packages/13/f0/358354786280a509482e0e77c1a5459e439766597d280f28cb097642fc26/Brotli-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87a3044c3a35055527ac75e419dfa9f4f3667a1e887ee80360589eb8c90aabb9", size = 2936146 }, + { url = "https://files.pythonhosted.org/packages/80/f7/daf538c1060d3a88266b80ecc1d1c98b79553b3f117a485653f17070ea2a/Brotli-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c5529b34c1c9d937168297f2c1fde7ebe9ebdd5e121297ff9c043bdb2ae3d6fb", size = 2848055 }, + { url = "https://files.pythonhosted.org/packages/ad/cf/0eaa0585c4077d3c2d1edf322d8e97aabf317941d3a72d7b3ad8bce004b0/Brotli-1.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ca63e1890ede90b2e4454f9a65135a4d387a4585ff8282bb72964fab893f2111", size = 3035102 }, + { url = "https://files.pythonhosted.org/packages/d8/63/1c1585b2aa554fe6dbce30f0c18bdbc877fa9a1bf5ff17677d9cca0ac122/Brotli-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e79e6520141d792237c70bcd7a3b122d00f2613769ae0cb61c52e89fd3443839", size = 2930029 }, + { url = "https://files.pythonhosted.org/packages/5f/3b/4e3fd1893eb3bbfef8e5a80d4508bec17a57bb92d586c85c12d28666bb13/Brotli-1.1.0-cp312-cp312-win32.whl", hash = "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0", size = 333276 }, + { url = "https://files.pythonhosted.org/packages/3d/d5/942051b45a9e883b5b6e98c041698b1eb2012d25e5948c58d6bf85b1bb43/Brotli-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951", size = 357255 }, + { url = "https://files.pythonhosted.org/packages/0a/9f/fb37bb8ffc52a8da37b1c03c459a8cd55df7a57bdccd8831d500e994a0ca/Brotli-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8bf32b98b75c13ec7cf774164172683d6e7891088f6316e54425fde1efc276d5", size = 815681 }, + { url = "https://files.pythonhosted.org/packages/06/b3/dbd332a988586fefb0aa49c779f59f47cae76855c2d00f450364bb574cac/Brotli-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bc37c4d6b87fb1017ea28c9508b36bbcb0c3d18b4260fcdf08b200c74a6aee8", size = 422475 }, + { url = "https://files.pythonhosted.org/packages/bb/80/6aaddc2f63dbcf2d93c2d204e49c11a9ec93a8c7c63261e2b4bd35198283/Brotli-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c0ef38c7a7014ffac184db9e04debe495d317cc9c6fb10071f7fefd93100a4f", size = 2906173 }, + { url = "https://files.pythonhosted.org/packages/ea/1d/e6ca79c96ff5b641df6097d299347507d39a9604bde8915e76bf026d6c77/Brotli-1.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91d7cc2a76b5567591d12c01f019dd7afce6ba8cba6571187e21e2fc418ae648", size = 2943803 }, + { url = "https://files.pythonhosted.org/packages/ac/a3/d98d2472e0130b7dd3acdbb7f390d478123dbf62b7d32bda5c830a96116d/Brotli-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a93dde851926f4f2678e704fadeb39e16c35d8baebd5252c9fd94ce8ce68c4a0", size = 2918946 }, + { url = "https://files.pythonhosted.org/packages/c4/a5/c69e6d272aee3e1423ed005d8915a7eaa0384c7de503da987f2d224d0721/Brotli-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0db75f47be8b8abc8d9e31bc7aad0547ca26f24a54e6fd10231d623f183d089", size = 2845707 }, + { url = "https://files.pythonhosted.org/packages/58/9f/4149d38b52725afa39067350696c09526de0125ebfbaab5acc5af28b42ea/Brotli-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6967ced6730aed543b8673008b5a391c3b1076d834ca438bbd70635c73775368", size = 2936231 }, + { url = "https://files.pythonhosted.org/packages/5a/5a/145de884285611838a16bebfdb060c231c52b8f84dfbe52b852a15780386/Brotli-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7eedaa5d036d9336c95915035fb57422054014ebdeb6f3b42eac809928e40d0c", size = 2848157 }, + { url = "https://files.pythonhosted.org/packages/50/ae/408b6bfb8525dadebd3b3dd5b19d631da4f7d46420321db44cd99dcf2f2c/Brotli-1.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d487f5432bf35b60ed625d7e1b448e2dc855422e87469e3f450aa5552b0eb284", size = 3035122 }, + { url = "https://files.pythonhosted.org/packages/af/85/a94e5cfaa0ca449d8f91c3d6f78313ebf919a0dbd55a100c711c6e9655bc/Brotli-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832436e59afb93e1836081a20f324cb185836c617659b07b129141a8426973c7", size = 2930206 }, + { url = "https://files.pythonhosted.org/packages/c2/f0/a61d9262cd01351df22e57ad7c34f66794709acab13f34be2675f45bf89d/Brotli-1.1.0-cp313-cp313-win32.whl", hash = "sha256:43395e90523f9c23a3d5bdf004733246fba087f2948f87ab28015f12359ca6a0", size = 333804 }, + { url = "https://files.pythonhosted.org/packages/7e/c1/ec214e9c94000d1c1974ec67ced1c970c148aa6b8d8373066123fc3dbf06/Brotli-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:9011560a466d2eb3f5a6e4929cf4a09be405c64154e12df0dd72713f6500e32b", size = 358517 }, + { url = "https://files.pythonhosted.org/packages/1b/aa/aa6e0c9848ee4375514af0b27abf470904992939b7363ae78fc8aca8a9a8/Brotli-1.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5fb2ce4b8045c78ebbc7b8f3c15062e435d47e7393cc57c25115cfd49883747a", size = 873048 }, + { url = "https://files.pythonhosted.org/packages/ae/32/38bba1a8bef9ecb1cda08439fd28d7e9c51aff13b4783a4f1610da90b6c2/Brotli-1.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7905193081db9bfa73b1219140b3d315831cbff0d8941f22da695832f0dd188f", size = 446207 }, + { url = "https://files.pythonhosted.org/packages/3c/6a/14cc20ddc53efc274601c8195791a27cfb7acc5e5134e0f8c493a8b8821a/Brotli-1.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a77def80806c421b4b0af06f45d65a136e7ac0bdca3c09d9e2ea4e515367c7e9", size = 2903803 }, + { url = "https://files.pythonhosted.org/packages/9a/26/62b2d894d4e82d7a7f4e0bb9007a42bbc765697a5679b43186acd68d7a79/Brotli-1.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8dadd1314583ec0bf2d1379f7008ad627cd6336625d6679cf2f8e67081b83acf", size = 2941149 }, + { url = "https://files.pythonhosted.org/packages/a9/ca/00d55bbdd8631236c61777742d8a8454cf6a87eb4125cad675912c68bec7/Brotli-1.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:901032ff242d479a0efa956d853d16875d42157f98951c0230f69e69f9c09bac", size = 2672253 }, + { url = "https://files.pythonhosted.org/packages/e2/e6/4a730f6e5b5d538e92d09bc51bf69119914f29a222f9e1d65ae4abb27a4e/Brotli-1.1.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:22fc2a8549ffe699bfba2256ab2ed0421a7b8fadff114a3d201794e45a9ff578", size = 2757005 }, + { url = "https://files.pythonhosted.org/packages/cb/6b/8cf297987fe3c1bf1c87f0c0b714af2ce47092b8d307b9f6ecbc65f98968/Brotli-1.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ae15b066e5ad21366600ebec29a7ccbc86812ed267e4b28e860b8ca16a2bc474", size = 2910658 }, + { url = "https://files.pythonhosted.org/packages/2c/1f/be9443995821c933aad7159803f84ef4923c6f5b72c2affd001192b310fc/Brotli-1.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:949f3b7c29912693cee0afcf09acd6ebc04c57af949d9bf77d6101ebb61e388c", size = 2809728 }, + { url = "https://files.pythonhosted.org/packages/76/2f/213bab6efa902658c80a1247142d42b138a27ccdd6bade49ca9cd74e714a/Brotli-1.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:89f4988c7203739d48c6f806f1e87a1d96e0806d44f0fba61dba81392c9e474d", size = 2935043 }, + { url = "https://files.pythonhosted.org/packages/27/89/bbb14fa98e895d1e601491fba54a5feec167d262f0d3d537a3b0d4cd0029/Brotli-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:de6551e370ef19f8de1807d0a9aa2cdfdce2e85ce88b122fe9f6b2b076837e59", size = 2930639 }, + { url = "https://files.pythonhosted.org/packages/14/87/03a6d6e1866eddf9f58cc57e35befbeb5514da87a416befe820150cae63f/Brotli-1.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0737ddb3068957cf1b054899b0883830bb1fec522ec76b1098f9b6e0f02d9419", size = 2932834 }, + { url = "https://files.pythonhosted.org/packages/a4/d5/e5f85e04f75144d1a89421ba432def6bdffc8f28b04f5b7d540bbd03362c/Brotli-1.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4f3607b129417e111e30637af1b56f24f7a49e64763253bbc275c75fa887d4b2", size = 2845213 }, + { url = "https://files.pythonhosted.org/packages/99/bf/25ef07add7afbb1aacd4460726a1a40370dfd60c0810b6f242a6d3871d7e/Brotli-1.1.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:6c6e0c425f22c1c719c42670d561ad682f7bfeeef918edea971a79ac5252437f", size = 3031573 }, + { url = "https://files.pythonhosted.org/packages/55/22/948a97bda5c9dc9968d56b9ed722d9727778db43739cf12ef26ff69be94d/Brotli-1.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:494994f807ba0b92092a163a0a283961369a65f6cbe01e8891132b7a320e61eb", size = 2926885 }, + { url = "https://files.pythonhosted.org/packages/31/ba/e53d107399b535ef89deb6977dd8eae468e2dde7b1b74c6cbe2c0e31fda2/Brotli-1.1.0-cp39-cp39-win32.whl", hash = "sha256:f0d8a7a6b5983c2496e364b969f0e526647a06b075d034f3297dc66f3b360c64", size = 333171 }, + { url = "https://files.pythonhosted.org/packages/99/b3/f7b3af539f74b82e1c64d28685a5200c631cc14ae751d37d6ed819655627/Brotli-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:cdad5b9014d83ca68c25d2e9444e28e967ef16e80f6b436918c700c117a85467", size = 357258 }, +] + +[[package]] +name = "brotlicffi" +version = "1.1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/9d/70caa61192f570fcf0352766331b735afa931b4c6bc9a348a0925cc13288/brotlicffi-1.1.0.0.tar.gz", hash = "sha256:b77827a689905143f87915310b93b273ab17888fd43ef350d4832c4a71083c13", size = 465192 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/11/7b96009d3dcc2c931e828ce1e157f03824a69fb728d06bfd7b2fc6f93718/brotlicffi-1.1.0.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9b7ae6bd1a3f0df532b6d67ff674099a96d22bc0948955cb338488c31bfb8851", size = 453786 }, + { url = "https://files.pythonhosted.org/packages/d6/e6/a8f46f4a4ee7856fbd6ac0c6fb0dc65ed181ba46cd77875b8d9bbe494d9e/brotlicffi-1.1.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19ffc919fa4fc6ace69286e0a23b3789b4219058313cf9b45625016bf7ff996b", size = 2911165 }, + { url = "https://files.pythonhosted.org/packages/be/20/201559dff14e83ba345a5ec03335607e47467b6633c210607e693aefac40/brotlicffi-1.1.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9feb210d932ffe7798ee62e6145d3a757eb6233aa9a4e7db78dd3690d7755814", size = 2927895 }, + { url = "https://files.pythonhosted.org/packages/cd/15/695b1409264143be3c933f708a3f81d53c4a1e1ebbc06f46331decbf6563/brotlicffi-1.1.0.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84763dbdef5dd5c24b75597a77e1b30c66604725707565188ba54bab4f114820", size = 2851834 }, + { url = "https://files.pythonhosted.org/packages/b4/40/b961a702463b6005baf952794c2e9e0099bde657d0d7e007f923883b907f/brotlicffi-1.1.0.0-cp37-abi3-win32.whl", hash = "sha256:1b12b50e07c3911e1efa3a8971543e7648100713d4e0971b13631cce22c587eb", size = 341731 }, + { url = "https://files.pythonhosted.org/packages/1c/fa/5408a03c041114ceab628ce21766a4ea882aa6f6f0a800e04ee3a30ec6b9/brotlicffi-1.1.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:994a4f0681bb6c6c3b0925530a1926b7a189d878e6e5e38fae8efa47c5d9c613", size = 366783 }, + { url = "https://files.pythonhosted.org/packages/e5/3b/bd4f3d2bcf2306ae66b0346f5b42af1962480b200096ffc7abc3bd130eca/brotlicffi-1.1.0.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:2e4aeb0bd2540cb91b069dbdd54d458da8c4334ceaf2d25df2f4af576d6766ca", size = 397397 }, + { url = "https://files.pythonhosted.org/packages/54/10/1fd57864449360852c535c2381ee7120ba8f390aa3869df967c44ca7eba1/brotlicffi-1.1.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b7b0033b0d37bb33009fb2fef73310e432e76f688af76c156b3594389d81391", size = 379698 }, + { url = "https://files.pythonhosted.org/packages/e5/95/15aa422aa6450e6556e54a5fd1650ff59f470aed77ac739aa90ab63dc611/brotlicffi-1.1.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54a07bb2374a1eba8ebb52b6fafffa2afd3c4df85ddd38fcc0511f2bb387c2a8", size = 378635 }, + { url = "https://files.pythonhosted.org/packages/6c/a7/f254e13b2cb43337d6d99a4ec10394c134e41bfda8a2eff15b75627f4a3d/brotlicffi-1.1.0.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7901a7dc4b88f1c1475de59ae9be59799db1007b7d059817948d8e4f12e24e35", size = 385719 }, + { url = "https://files.pythonhosted.org/packages/72/a9/0971251c4427c14b2a827dba3d910d4d3330dabf23d4278bf6d06a978847/brotlicffi-1.1.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ce01c7316aebc7fce59da734286148b1d1b9455f89cf2c8a4dfce7d41db55c2d", size = 361760 }, + { url = "https://files.pythonhosted.org/packages/35/9b/e0b577351e1d9d5890e1a56900c4ceaaef783b807145cd229446a43cf437/brotlicffi-1.1.0.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1a807d760763e398bbf2c6394ae9da5815901aa93ee0a37bca5efe78d4ee3171", size = 397392 }, + { url = "https://files.pythonhosted.org/packages/4f/7f/a16534d28386f74781db8b4544a764cf955abae336379a76f50e745bb0ee/brotlicffi-1.1.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fa8ca0623b26c94fccc3a1fdd895be1743b838f3917300506d04aa3346fd2a14", size = 379695 }, + { url = "https://files.pythonhosted.org/packages/50/2a/699388b5e489726991132441b55aff0691dd73c49105ef220408a5ab98d6/brotlicffi-1.1.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3de0cf28a53a3238b252aca9fed1593e9d36c1d116748013339f0949bfc84112", size = 378629 }, + { url = "https://files.pythonhosted.org/packages/4a/3f/58254e7fbe6011bf043e4dcade0e16995a9f82b731734fad97220d201f42/brotlicffi-1.1.0.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6be5ec0e88a4925c91f3dea2bb0013b3a2accda6f77238f76a34a1ea532a1cb0", size = 385712 }, + { url = "https://files.pythonhosted.org/packages/40/16/2a29a625a6f74d13726387f83484dfaaf6fcdaafaadfbe26a0412ae268cc/brotlicffi-1.1.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d9eb71bb1085d996244439154387266fd23d6ad37161f6f52f1cd41dd95a3808", size = 361747 }, +] + [[package]] name = "cachetools" version = "5.5.0" @@ -687,6 +800,54 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/d9/a1e041c5e7caa9a05c925f4bdbdfb7f006d1f74996af53467bc394c97be7/importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b", size = 26514 }, ] +[[package]] +name = "inflate64" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/99/18f9940d4a3f2cabc4396a587ddf1bd93236bdb372d9e78e2b0365e40990/inflate64-1.0.0.tar.gz", hash = "sha256:3278827b803cf006a1df251f3e13374c7d26db779e5a33329cc11789b804bc2d", size = 895853 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/cf/06af80e81dd4bbb7e883291cf1726035d526f066a37c4ed4d4cd88a7a49d/inflate64-1.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a90c0bdf4a7ecddd8a64cc977181810036e35807f56b0bcacee9abb0fcfd18dc", size = 59418 }, + { url = "https://files.pythonhosted.org/packages/c9/4b/6f18918220b1a8e935121cece1dc917e62fa593fc637a621470f9b9a601a/inflate64-1.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:57fe7c14aebf1c5a74fc3b70d355be1280a011521a76aa3895486e62454f4242", size = 36231 }, + { url = "https://files.pythonhosted.org/packages/aa/f4/f4b5dbd78dd5af66b6ca32778ebaa9c14d67b68ea84e96592ccf40786a41/inflate64-1.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d90730165f471d61a1a694a5e354f3ffa938227e8dcecb62d5d728e8069cee94", size = 35738 }, + { url = "https://files.pythonhosted.org/packages/10/23/26289a700550767cf5eb7550f78ad826529706287393f224bbaee3c1b1e2/inflate64-1.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:543f400201f5c101141af3c79c82059e1aa6ef4f1584a7f1fa035fb2e465097f", size = 92855 }, + { url = "https://files.pythonhosted.org/packages/b8/f4/e387a50f5027194eac4f9712d57b97e3e1a012402eaae98bcf1ebe8a97d1/inflate64-1.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ceca14f7ec19fb44b047f56c50efb7521b389d222bba2b0a10286a0caeb03fa", size = 93141 }, + { url = "https://files.pythonhosted.org/packages/33/c8/e516aecd9ed0dc75d8df041ed4ef80f2e2be39d0e516c7269b7f274e760a/inflate64-1.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b559937a42f0c175b4d2dfc7eb53b97bdc87efa9add15ed5549c6abc1e89d02f", size = 95262 }, + { url = "https://files.pythonhosted.org/packages/0b/aa/ed3ab5f8c13afc432fb382edf97cede7a6f9be73ecf98bfe64b686c8d223/inflate64-1.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5ff8bd2a562343fcbc4eea26fdc368904a3b5f6bb8262344274d3d74a1de15bb", size = 95912 }, + { url = "https://files.pythonhosted.org/packages/e0/64/5637c4f67ed15518c0765b85b528ed79536caaf8ba167a9f7173e334d4a8/inflate64-1.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:0fe481f31695d35a433c3044ac8fd5d9f5069aaad03a0c04b570eb258ce655aa", size = 35166 }, + { url = "https://files.pythonhosted.org/packages/af/92/701b3c76b1cf244026c3e78dff8487955cf6960c1d9f350e2820a0d1a5d9/inflate64-1.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a45f6979ad5874d4d4898c2fc770b136e61b96b850118fdaec5a5af1b9123a", size = 59450 }, + { url = "https://files.pythonhosted.org/packages/bb/1d/af0253fafc27cadd29e3b111ebb3011b8c913a3554b403c90c7595f5933e/inflate64-1.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:022ca1cc928e7365a05f7371ff06af143c6c667144965e2cf9a9236a2ae1c291", size = 36267 }, + { url = "https://files.pythonhosted.org/packages/b6/22/7949030be11f4754bd6ed7067e9bebdf614013b89ccd4638330a85821b51/inflate64-1.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:46792ecf3565d64fd2c519b0a780c03a57e195613c9954ef94e739a057b3fd06", size = 35740 }, + { url = "https://files.pythonhosted.org/packages/e4/87/c6ce0093a345c04811f6171a367665dec17dcc4617ca150dd37e9ae7bd33/inflate64-1.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a70ea2e456c15f7aa7c74b8ab8f20b4f8940ec657604c9f0a9de3342f280fff", size = 95896 }, + { url = "https://files.pythonhosted.org/packages/62/d6/fe113b12773cad2c093d381c2b1629f9cfa240c9ad86a7f9f9079e7a51b5/inflate64-1.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e243ea9bd36a035059f2365bd6d156ff59717fbafb0255cb0c75bf151bf6904", size = 96007 }, + { url = "https://files.pythonhosted.org/packages/f0/a6/9165bee4b7fc5af949fec12a2cea7ad73bf9ee97dfb96a0276274c48e709/inflate64-1.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4dc392dec1cd11cacda3d2637214ca45e38202e8a4f31d4a4e566d6e90625fc4", size = 98297 }, + { url = "https://files.pythonhosted.org/packages/ee/72/0aeb360101eeed32696fc6c623bc1780fac895a9fc2e93b582cb1e22ca54/inflate64-1.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8b402a50eda7ee75f342fc346d33a41bca58edc222a4b17f9be0db1daed459fa", size = 98858 }, + { url = "https://files.pythonhosted.org/packages/94/4a/8301ad59b57d9de504b0fdce22bf980dfb231753e6d7aed12af938f7f9fd/inflate64-1.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:f5924499dc8800928c0ee4580fa8eb4ffa880b2cce4431537d0390e503a9c9ee", size = 35167 }, + { url = "https://files.pythonhosted.org/packages/18/82/47021b8919c1dc276d0502296f15ffac1cd648b94b35cadb14cb812b6199/inflate64-1.0.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0c644bf7208e20825ca3bbb5fb1f7f495cfcb49eb01a5f67338796d44a42f2bf", size = 59509 }, + { url = "https://files.pythonhosted.org/packages/e0/c9/00701be8e48dc9c9b9488001d9c66d6cb6f6bb0c48af9abf33a69726d130/inflate64-1.0.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9964a4eaf26a9d36f82a1d9b12c28e35800dd3d99eb340453ed12ac90c2976a8", size = 36305 }, + { url = "https://files.pythonhosted.org/packages/25/c0/11dea5e298b2e7d61f0fbd1005553e8796e35536751980b676547fcc57ef/inflate64-1.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2cccded63865640d03253897be7232b2bbac295fe43914c61f86a57aa23bb61d", size = 35756 }, + { url = "https://files.pythonhosted.org/packages/86/ba/4debdaaafdc21853621caf463a498a754ee4352893454c596dbd65294e9f/inflate64-1.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d491f104fb3701926ebd82b8c9250dfba0ddcab584504e26f1e4adb26730378d", size = 96127 }, + { url = "https://files.pythonhosted.org/packages/89/81/8f559c199ec13d0b70d0dc46811490b2976873c96c564941583777e9b343/inflate64-1.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ebad4a6cd2a2c1d81be0b09d4006479f3b258803c49a9224ef8ca0b649072fa", size = 96903 }, + { url = "https://files.pythonhosted.org/packages/46/41/39ac4c7e17d0690578b716a0ff34e00600616994795b0645fd61fc600c0f/inflate64-1.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:6823b2c0cff3a8159140f3b17ec64fb8ec0e663b45a6593618ecdde8aeecb5b2", size = 98855 }, + { url = "https://files.pythonhosted.org/packages/44/dd/be5d69492c180f94a6af8a15564ce365bdcb84bd1a6fb32949d6913959aa/inflate64-1.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:228d504239d27958e71fc77e3119a6ac4528127df38468a0c95a5bd3927204b8", size = 99884 }, + { url = "https://files.pythonhosted.org/packages/8c/0d/a5266bd4f2cdb7fad1eae3ffe4dcc16f9769323660a0a6cfbe9cc1d2cf03/inflate64-1.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:ae2572e06bcfe15e3bbf77d4e4a6d6c55e2a70d6abceaaf60c5c3653ddb96dfd", size = 35334 }, + { url = "https://files.pythonhosted.org/packages/04/1c/47f9c93df339c381f3f3e7b983d7abf0756f8bd227bf9fb5a2e8b09ea9e5/inflate64-1.0.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bf2981b95c1f26242bb084d9a07f3feb0cfe3d6d0a8d90f42389803bc1252c4a", size = 59410 }, + { url = "https://files.pythonhosted.org/packages/b9/28/bde1595cae0379c521ae74f698f7da1345d990d2df7558df59470ced639b/inflate64-1.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9373ccf0661cc72ac84a0ad622634144da5ce7d57c9572ed0723d67a149feed2", size = 36230 }, + { url = "https://files.pythonhosted.org/packages/13/64/4350547de7c8a3ac27c1a0ab5807ac2fcbcde121c44190f87da1039421eb/inflate64-1.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e4650c6f65011ec57cf5cd96b92d5b7c6f59e502930c86eb8227c93cf02dc270", size = 35739 }, + { url = "https://files.pythonhosted.org/packages/1a/d6/b55e43c3503d7780aabe3d1d5493c983a99947dff228d9b622d49c58af68/inflate64-1.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a475e8822f1a74c873e60b8f270773757ade024097ca39e43402d47c049c67d4", size = 92655 }, + { url = "https://files.pythonhosted.org/packages/c0/c0/417e5183543445818930b3fe181d718e519d26a227b5b77871d8f0c8502d/inflate64-1.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4367480733ac8daf368f6fc704b7c9db85521ee745eb5bd443f4b97d2051acc", size = 92966 }, + { url = "https://files.pythonhosted.org/packages/6d/8e/ada659c83abb78222c666bb8d35b4791cca25a8a4a750f4bc457402d2430/inflate64-1.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6c5775c91f94f5eced9160fb0af12a09f3e030194f91a6a46e706a79350bd056", size = 95075 }, + { url = "https://files.pythonhosted.org/packages/7f/15/59c0e45e091599a05a098374166ff74d3da5f9c0ffa72e8c78b7f0fcaa2b/inflate64-1.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d76d205b844d78ce04768060084ef20e64dcc63a3e9166674f857acaf4d140ed", size = 95751 }, + { url = "https://files.pythonhosted.org/packages/80/21/a90d085b9ea67729d5cef57566c69fd4dec8205c22415cde4be62b7adc64/inflate64-1.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:92f0dc6af0e8e97324981178dc442956cbff1247a56d1e201af8d865244653f8", size = 35164 }, + { url = "https://files.pythonhosted.org/packages/53/91/43238dd8a7e5bab71abae872c09931db4b31aebf672afccb305f79aacb3e/inflate64-1.0.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f79542478e49e471e8b23556700e6f688a40dc93e9a746f77a546c13251b59b1", size = 34648 }, + { url = "https://files.pythonhosted.org/packages/ef/6f/ce090934a80c1fd0b5b07c125ed6eb2845f11a78af344d69c0f051dcab97/inflate64-1.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a270be6b10cde01258c0097a663a307c62d12c78eb8f62f8e29f205335942c9", size = 36473 }, + { url = "https://files.pythonhosted.org/packages/b4/fe/2cd4bf78696213b807860002c182dd1751ba52c1559143b1b8daa7904733/inflate64-1.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1616a87ff04f583e9558cc247ec0b72a30d540ee0c17cc77823be175c0ec92f0", size = 36478 }, + { url = "https://files.pythonhosted.org/packages/43/dd/e62444c0ef7d1228b622e6d3dacf9ea237d8807a78619a83832a3b4a5adf/inflate64-1.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:137ca6b315f0157a786c3a755a09395ca69aed8bcf42ad3437cb349f5ebc86d2", size = 35630 }, + { url = "https://files.pythonhosted.org/packages/2b/1e/0e346cb5eced6f2f30bb110e6d7c7ee7ab70ad1ff44b743d4cf314f60f7d/inflate64-1.0.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:dd6d3e7d47df43210a995fd1f5989602b64de3f2a17cf4cbff553518b3577fd4", size = 34644 }, + { url = "https://files.pythonhosted.org/packages/d4/f7/7ac502391ea56af23661a707e9c94efc01376a27c1b06f7a0cfb553c0f17/inflate64-1.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f033b2879696b855200cde5ca4e293132c7499df790acb2c0dacb336d5e83b1", size = 36467 }, + { url = "https://files.pythonhosted.org/packages/09/ea/7c3e7c856f42bee178a9fea816e259b02bd4198873cf07b5a839b96d9696/inflate64-1.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f816d1c8a0593375c289e285c96deaee9c2d8742cb0edbd26ee05588a9ae657", size = 36475 }, + { url = "https://files.pythonhosted.org/packages/ae/b1/954207a9bd259a57c8c71ba0a26739a32f0d6227529abc9412751f3a87d2/inflate64-1.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1facd35319b6a391ee4c3d709c7c650bcada8cd7141d86cd8c2257287f45e6e6", size = 35626 }, +] + [[package]] name = "iniconfig" version = "2.0.0" @@ -1225,6 +1386,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f0/74/c95adcdf032956d9ef6c89a9b8a5152bf73915f8c633f3e3d88d06bd699c/mistune-3.0.2-py3-none-any.whl", hash = "sha256:71481854c30fdbc938963d3605b72501f5c10a9320ecd412c121c163a1c7d205", size = 47958 }, ] +[[package]] +name = "multivolumefile" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/50/f0/a7786212b5a4cb9ba05ae84a2bbd11d1d0279523aea0424b6d981d652a14/multivolumefile-0.2.3.tar.gz", hash = "sha256:a0648d0aafbc96e59198d5c17e9acad7eb531abea51035d08ce8060dcad709d6", size = 77984 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/31/ec5f46fd4c83185b806aa9c736e228cb780f13990a9cf4da0beb70025fcc/multivolumefile-0.2.3-py3-none-any.whl", hash = "sha256:237f4353b60af1703087cf7725755a1f6fcaeeea48421e1896940cd1c920d678", size = 17037 }, +] + [[package]] name = "mypy-extensions" version = "1.0.0" @@ -1428,6 +1598,7 @@ dependencies = [ { name = "gdown" }, { name = "gsheets" }, { name = "pandas" }, + { name = "py7zr" }, { name = "pyarrow" }, { name = "pydrive2" }, { name = "structlog" }, @@ -1442,6 +1613,7 @@ requires-dist = [ { name = "gdown", specifier = ">=4.5.2" }, { name = "gsheets", specifier = ">=0.6.1" }, { name = "pandas", specifier = ">=2.2.1" }, + { name = "py7zr", specifier = ">=0.22.0" }, { name = "pyarrow", specifier = ">=10.0.1" }, { name = "pydrive2", specifier = ">=1.15.0" }, { name = "structlog", specifier = ">=21.5.0" }, @@ -1460,7 +1632,7 @@ dev = [ { name = "pyright", specifier = "==1.1.373" }, { name = "pytest", specifier = ">=6.2.5" }, { name = "pytest-cov", specifier = ">=2.12.1" }, - { name = "ruff", specifier = "==0.1.6" }, + { name = "ruff", specifier = "==0.8.2" }, { name = "sphinx", specifier = "==4.5.0" }, { name = "watchdog", specifier = ">=2.1.5" }, ] @@ -1487,7 +1659,7 @@ dev = [ { name = "ipdb", specifier = ">=0.13.13" }, { name = "pyright", specifier = "==1.1.373" }, { name = "pytest", specifier = ">=7.2.0" }, - { name = "ruff", specifier = "==0.1.6" }, + { name = "ruff", specifier = "==0.8.2" }, ] [[package]] @@ -1686,6 +1858,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842 }, ] +[[package]] +name = "py7zr" +version = "0.22.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "brotli", marker = "platform_python_implementation == 'CPython'" }, + { name = "brotlicffi", marker = "platform_python_implementation == 'PyPy'" }, + { name = "inflate64" }, + { name = "multivolumefile" }, + { name = "psutil", marker = "sys_platform != 'cygwin'" }, + { name = "pybcj" }, + { name = "pycryptodomex" }, + { name = "pyppmd" }, + { name = "pyzstd" }, + { name = "texttable" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/c3/0e05c711c16af0b9c47f3f77323303b338b9a871ba020d95d2b8dd6605ae/py7zr-0.22.0.tar.gz", hash = "sha256:c6c7aea5913535184003b73938490f9a4d8418598e533f9ca991d3b8e45a139e", size = 4992926 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/59/dd1750002c0f46099281116f8165247bc62dc85edad41cdd26e7b26de19d/py7zr-0.22.0-py3-none-any.whl", hash = "sha256:993b951b313500697d71113da2681386589b7b74f12e48ba13cc12beca79d078", size = 67906 }, +] + [[package]] name = "pyarrow" version = "17.0.0" @@ -1746,6 +1939,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/89/bc88a6711935ba795a679ea6ebee07e128050d6382eaa35a0a47c8032bdc/pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd", size = 181537 }, ] +[[package]] +name = "pybcj" +version = "1.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/d2/22e808b9d25ce3b43f5c8a9e22d873d403485ba55d84a4d6d5d044881762/pybcj-1.0.2.tar.gz", hash = "sha256:c7f5bef7f47723c53420e377bc64d2553843bee8bcac5f0ad076ab1524780018", size = 2111002 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/93/4735636b5905b7597068a2c7a10a8df0f668f28659207c274d64a4468b97/pybcj-1.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7bff28d97e47047d69a4ac6bf59adda738cf1d00adde8819117fdb65d966bdbc", size = 32556 }, + { url = "https://files.pythonhosted.org/packages/a6/37/443cd704397b6df54ff0822032e4815aca4e9badabc5ce1faac34235a40c/pybcj-1.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:198e0b4768b4025eb3309273d7e81dc53834b9a50092be6e0d9b3983cfd35c35", size = 23751 }, + { url = "https://files.pythonhosted.org/packages/9a/aa/5a19ed8661e979a4d3237a11706f9a16a474a2227fdd99ccb284be100a98/pybcj-1.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fa26415b4a118ea790de9d38f244312f2510a9bb5c65e560184d241a6f391a2d", size = 23980 }, + { url = "https://files.pythonhosted.org/packages/fe/5f/638ce03948905d267c8c0ccab81b8b4943a0324f63d8bdb0a0e2a85d4503/pybcj-1.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fabb2be57e4ca28ea36c13146cdf97d73abd27c51741923fc6ba1e8cd33e255c", size = 50155 }, + { url = "https://files.pythonhosted.org/packages/09/70/8b6a6cc2a5721f67f629bdc17875c0d603d57f360a19b099a7b4de19383d/pybcj-1.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75d6d613bae6f27678d5e44e89d61018779726aa6aa950c516d33a04b8af8c59", size = 49729 }, + { url = "https://files.pythonhosted.org/packages/89/06/2e41e34da0bb2adb3644cbf4366c344e5804a10f1153da7b3a23333f7db8/pybcj-1.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3ffae79ef8a1ea81ea2748ad7b7ad9b882aa88ddf65ce90f9e944df639eccc61", size = 54310 }, + { url = "https://files.pythonhosted.org/packages/b5/0f/de9e76c305d4dcd9d428a90ccac030f06c780bc30549fc449a944a6321bc/pybcj-1.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bdb4d8ff5cba3e0bd1adee7d20dbb2b4d80cb31ac04d6ea1cd06cfc02d2ecd0d", size = 53679 }, + { url = "https://files.pythonhosted.org/packages/1a/41/a807ff6b77ec8e49c749ed1d0db5649fbb1150c6fb5fb391115f4f1d743a/pybcj-1.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a29be917fbc99eca204b08407e0971e0205bfdad4b74ec915930675f352b669d", size = 24690 }, + { url = "https://files.pythonhosted.org/packages/27/0a/20bf70a7eb7c6b2668ff2af798254033c32a09d6c58ec9a87cd6aa843df5/pybcj-1.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a2562ebe5a0abec4da0229f8abb5e90ee97b178f19762eb925c1159be36828b3", size = 32581 }, + { url = "https://files.pythonhosted.org/packages/a9/b6/43977fe4296d2778c6dc67b596bb6a851eaea80f3dd4ff454e5fca8142c2/pybcj-1.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:af19bc61ded933001cd68f004ae2042bf1a78eb498a3c685ebd655fa1be90dbe", size = 23767 }, + { url = "https://files.pythonhosted.org/packages/89/c7/a61010f59406b8a45bb4865faa4b61d6b177dcfac04247fb56c7538d997d/pybcj-1.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f3f4a447800850aba7724a2274ea0a4800724520c1caf38f7d0dabf2f89a5e15", size = 23976 }, + { url = "https://files.pythonhosted.org/packages/10/7a/78848edbb6f12d9b86e375fc46135d9a204ededbf96682b05cb4b4fbd942/pybcj-1.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce1c8af7a4761d2b1b531864d84113948daa0c4245775c63bd9874cb955f4662", size = 51246 }, + { url = "https://files.pythonhosted.org/packages/9e/13/af86c86cdfb293e82dd0b6c4bbdf08645cd8993456ee3fb911c3eeed1b22/pybcj-1.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8007371f6f2b462f5aa05d5c2135d0a1bcf5b7bdd9bd15d86c730f588d10b7d3", size = 50754 }, + { url = "https://files.pythonhosted.org/packages/39/52/88600aa374b100612a1d82fca4b03eb4315e0084a05ee314ba1b771f7190/pybcj-1.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1079ca63ff8da5c936b76863690e0bd2489e8d4e0a3a340e032095dae805dd91", size = 55334 }, + { url = "https://files.pythonhosted.org/packages/56/67/3cf9747ef5b53e16a844217c6c9840be6289d05ec785500da2cc55cc25f2/pybcj-1.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e9a785eb26884429d9b9f6326e68c3638828c83bf6d42d2463c97ad5385caff2", size = 54714 }, + { url = "https://files.pythonhosted.org/packages/78/81/a71197903b503f54b85f4d352f909e701e9d26953577bd34d3fbe0520d5d/pybcj-1.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:9ea46e2d45469d13b7f25b08efcdb140220bab1ac5a850db0954591715b8caaa", size = 24693 }, + { url = "https://files.pythonhosted.org/packages/83/60/a3b43836895654aa93b5a8422adc3717359db98da9147abfabffef79f1e7/pybcj-1.0.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:21b5f2460629167340403d359289a173e0729ce8e84e3ce99462009d5d5e01a4", size = 32677 }, + { url = "https://files.pythonhosted.org/packages/50/b9/96c8d9577b0f5a701e4497408e6a331a08eb902aca8dfd4c5bb1eaab4779/pybcj-1.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:2940fb85730b9869254559c491cd83cf777e56c76a8a60df60e4be4f2a4248d7", size = 23813 }, + { url = "https://files.pythonhosted.org/packages/b7/1a/c80132feb084ec4098c0315a132799bddda8878113b5f956e21c4377f5f1/pybcj-1.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f40f3243139d675f43793a4e35c410c370f7b91ccae74e70c8b2f4877869f90e", size = 24019 }, + { url = "https://files.pythonhosted.org/packages/b1/94/62c3bf8a60b4787b46e21f43277d9cb8b6037c8ee183450f035a19a2bc4b/pybcj-1.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c2b3e60b65c7ac73e44335934e1e122da8d56db87840984601b3c5dc0ae4c19", size = 51927 }, + { url = "https://files.pythonhosted.org/packages/8b/9e/4ebd092251ef8d15408388be508617d5949cbba4baa2a6cfbb7e0a9b62c0/pybcj-1.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:746550dc7b5af4d04bb5fa4d065f18d39c925bcb5dee30db75747cd9a58bb6e8", size = 51665 }, + { url = "https://files.pythonhosted.org/packages/24/ea/da4637563468854bd361a69cd883946015f54fa119a5d9c655d26f151954/pybcj-1.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:8ce9b62b6aaa5b08773be8a919ecc4e865396c969f982b685eeca6e80c82abb7", size = 56041 }, + { url = "https://files.pythonhosted.org/packages/cf/b2/9b9e670818af925ed9a0168a5c021ccfcc089637d0e6651d16fd05896425/pybcj-1.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:493eab2b1f6f546730a6de0c5ceb75ce16f3767154e8ae30e2b70d41b928b7d2", size = 55606 }, + { url = "https://files.pythonhosted.org/packages/72/e9/d6b1bdf3a5aca8f3981145a5228ad51d72e2477a55927604a4768765e915/pybcj-1.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:ef55b96b7f2ed823e0b924de902065ec42ade856366c287dbb073fabd6b90ec1", size = 24719 }, + { url = "https://files.pythonhosted.org/packages/6e/18/ca43a186a570b3f6820a24f3ad726d0f0322f0b08f5550a92f99741a2e58/pybcj-1.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:fdb7cd8271471a5979d84915c1ee57eea7e0a69c893225fc418db66883b0e2a7", size = 32548 }, + { url = "https://files.pythonhosted.org/packages/f1/10/ee383b3450f7f13fbe234668fcf143b9a1e916e0cb84f5267ff3a5c6ad60/pybcj-1.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e96ae14062bdcddc3197300e6ee4efa6fbc6749be917db934eac66d0daaecb68", size = 23743 }, + { url = "https://files.pythonhosted.org/packages/1f/90/2d2851d694a7dc3640c6309af0e534cab145e7463bc408114db7eaa20115/pybcj-1.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a54ebdc8423ba99d75372708a882fcfc3b14d9d52cf195295ad53e5a47dab37f", size = 23971 }, + { url = "https://files.pythonhosted.org/packages/9a/29/8326dbfea26d643d5a95f836103ac278eb297143d881188d94b987e3a520/pybcj-1.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3602be737c6e9553c45ae89e6b0e556f64f34dabf27d5260317d1824d31b79d3", size = 49937 }, + { url = "https://files.pythonhosted.org/packages/ac/b2/26fa2cba6bc488380515929757cafbdbf01f30184a1aa11ef7ee35bb21a2/pybcj-1.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63dd2ca52a48841f561bfec0fa3f208d375b0a8dcd3d7b236459e683ae29221d", size = 49517 }, + { url = "https://files.pythonhosted.org/packages/58/7b/d5e39a73202eb7b67793a4313ae5a85bdbf1470899dc2d3119c6a2414e9b/pybcj-1.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8204a714029784b1a08a3d790430d80b423b68615c5b1e67aabca5bd5419b77d", size = 54141 }, + { url = "https://files.pythonhosted.org/packages/8e/f0/5abc858fe9d07338e485e86b2d82d1f1a0aa36c2af5271156c1140d04d15/pybcj-1.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:fde2376b180ae2620c102fbc3ef06638d306feae83964aaa5051ecbdda54845a", size = 53499 }, + { url = "https://files.pythonhosted.org/packages/9f/ac/4bad26429aab693235035f813dc60ff00ff5164acbbb98c4e26f190a21cd/pybcj-1.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:3b8d7810fb587adbffba025330cf212d9bbed8f29559656d05cb6609673f306a", size = 24686 }, +] + [[package]] name = "pycparser" version = "2.22" @@ -1755,6 +1988,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552 }, ] +[[package]] +name = "pycryptodomex" +version = "3.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/dc/e66551683ade663b5f07d7b3bc46434bf703491dbd22ee12d1f979ca828f/pycryptodomex-3.21.0.tar.gz", hash = "sha256:222d0bd05381dd25c32dd6065c071ebf084212ab79bab4599ba9e6a3e0009e6c", size = 4818543 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/5e/99f217d9881eead69607a2248dd7bbdf610837d7f5ad53f45a6cb71bbbfb/pycryptodomex-3.21.0-cp36-abi3-macosx_10_9_universal2.whl", hash = "sha256:34325b84c8b380675fd2320d0649cdcbc9cf1e0d1526edbe8fce43ed858cdc7e", size = 2499490 }, + { url = "https://files.pythonhosted.org/packages/ce/8f/4d0e2a859a6470289d64e39b419f01d2494dfa2e4995342d50f6c2834237/pycryptodomex-3.21.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:103c133d6cd832ae7266feb0a65b69e3a5e4dbbd6f3a3ae3211a557fd653f516", size = 1638037 }, + { url = "https://files.pythonhosted.org/packages/0c/9e/6e748c1fa814c956d356f93cf7192b19487ca56fc9e2a0bcde2bbc057601/pycryptodomex-3.21.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77ac2ea80bcb4b4e1c6a596734c775a1615d23e31794967416afc14852a639d3", size = 2172279 }, + { url = "https://files.pythonhosted.org/packages/46/3f/f5bef92b11750af9e3516d4e69736eeeff20a2818d34611508bef5a7b381/pycryptodomex-3.21.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9aa0cf13a1a1128b3e964dc667e5fe5c6235f7d7cfb0277213f0e2a783837cc2", size = 2258130 }, + { url = "https://files.pythonhosted.org/packages/de/4d/f0c65afd64ce435fd0547187ce6f99dfb37cdde16b05b57bca9f5c06966e/pycryptodomex-3.21.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:46eb1f0c8d309da63a2064c28de54e5e614ad17b7e2f88df0faef58ce192fc7b", size = 2297719 }, + { url = "https://files.pythonhosted.org/packages/1c/6a/2a1a101b0345ee70376ba93df8de6c8c01aac8341fda02970800873456a7/pycryptodomex-3.21.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:cc7e111e66c274b0df5f4efa679eb31e23c7545d702333dfd2df10ab02c2a2ce", size = 2164079 }, + { url = "https://files.pythonhosted.org/packages/3d/00/90a15f16c234815b660303c2d7266b41b401ea2605f3a90373e9d425e39f/pycryptodomex-3.21.0-cp36-abi3-musllinux_1_2_i686.whl", hash = "sha256:770d630a5c46605ec83393feaa73a9635a60e55b112e1fb0c3cea84c2897aa0a", size = 2333060 }, + { url = "https://files.pythonhosted.org/packages/61/74/49f5d20c514ccc631b940cc9dfec45dcce418dc84a98463a2e2ebec33904/pycryptodomex-3.21.0-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:52e23a0a6e61691134aa8c8beba89de420602541afaae70f66e16060fdcd677e", size = 2257982 }, + { url = "https://files.pythonhosted.org/packages/92/4b/d33ef74e2cc0025a259936661bb53432c5bbbadc561c5f2e023bcd73ce4c/pycryptodomex-3.21.0-cp36-abi3-win32.whl", hash = "sha256:a3d77919e6ff56d89aada1bd009b727b874d464cb0e2e3f00a49f7d2e709d76e", size = 1779052 }, + { url = "https://files.pythonhosted.org/packages/5b/be/7c991840af1184009fc86267160948350d1bf875f153c97bb471ad944e40/pycryptodomex-3.21.0-cp36-abi3-win_amd64.whl", hash = "sha256:b0e9765f93fe4890f39875e6c90c96cb341767833cfa767f41b490b506fa9ec0", size = 1816307 }, + { url = "https://files.pythonhosted.org/packages/af/ac/24125ad36778914a36f08d61ba5338cb9159382c638d9761ee19c8de822c/pycryptodomex-3.21.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:feaecdce4e5c0045e7a287de0c4351284391fe170729aa9182f6bd967631b3a8", size = 1694999 }, + { url = "https://files.pythonhosted.org/packages/93/73/be7a54a5903508070e5508925ba94493a1f326cfeecfff750e3eb250ea28/pycryptodomex-3.21.0-pp27-pypy_73-win32.whl", hash = "sha256:365aa5a66d52fd1f9e0530ea97f392c48c409c2f01ff8b9a39c73ed6f527d36c", size = 1769437 }, + { url = "https://files.pythonhosted.org/packages/e5/9f/39a6187f3986841fa6a9f35c6fdca5030ef73ff708b45a993813a51d7d10/pycryptodomex-3.21.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3efddfc50ac0ca143364042324046800c126a1d63816d532f2e19e6f2d8c0c31", size = 1619607 }, + { url = "https://files.pythonhosted.org/packages/f8/70/60bb08e9e9841b18d4669fb69d84b64ce900aacd7eb0ebebd4c7b9bdecd3/pycryptodomex-3.21.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0df2608682db8279a9ebbaf05a72f62a321433522ed0e499bc486a6889b96bf3", size = 1653571 }, + { url = "https://files.pythonhosted.org/packages/c9/6f/191b73509291c5ff0dddec9cc54797b1d73303c12b2e4017b24678e57099/pycryptodomex-3.21.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5823d03e904ea3e53aebd6799d6b8ec63b7675b5d2f4a4bd5e3adcb512d03b37", size = 1691548 }, + { url = "https://files.pythonhosted.org/packages/2d/c7/a0d3356f3074ac548afefa515ff46f3bea011deca607faf1c09b26dd5330/pycryptodomex-3.21.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:27e84eeff24250ffec32722334749ac2a57a5fd60332cd6a0680090e7c42877e", size = 1792099 }, + { url = "https://files.pythonhosted.org/packages/55/ee/9349856ee02826899fdc489016756865158217909a82dcc74cc4d55d33af/pycryptodomex-3.21.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:8ef436cdeea794015263853311f84c1ff0341b98fc7908e8a70595a68cefd971", size = 1619490 }, + { url = "https://files.pythonhosted.org/packages/07/93/e68fac121fcf761fd7a85a27f024c9238217e4d943c861a856ca354f412e/pycryptodomex-3.21.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a1058e6dfe827f4209c5cae466e67610bcd0d66f2f037465daa2a29d92d952b", size = 1653481 }, + { url = "https://files.pythonhosted.org/packages/b8/47/8a39243d09fd294c339c59834ba3c92715584f3ed0d92b6bacb26f803ce0/pycryptodomex-3.21.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9ba09a5b407cbb3bcb325221e346a140605714b5e880741dc9a1e9ecf1688d42", size = 1691442 }, + { url = "https://files.pythonhosted.org/packages/21/1c/f8860c558b44776573acd719c1e86fec14d42f29cf248eaba9c770151d14/pycryptodomex-3.21.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:8a9d8342cf22b74a746e3c6c9453cb0cfbb55943410e3a2619bd9164b48dc9d9", size = 1791966 }, +] + [[package]] name = "pydrive2" version = "1.20.0" @@ -1800,6 +2061,68 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/0c/0e3c05b1c87bb6a1c76d281b0f35e78d2d80ac91b5f8f524cebf77f51049/pyparsing-3.1.4-py3-none-any.whl", hash = "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c", size = 104100 }, ] +[[package]] +name = "pyppmd" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/39/c8/9039c7503577de08a3f4c81e7619583efdc16030da6d1a25268d3dca49c8/pyppmd-1.1.0.tar.gz", hash = "sha256:1d38ce2e4b7eb84b53bc8a52380b94f66ba6c39328b8800b30c2b5bf31693973", size = 1348949 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/10/b19621035862e2ae12a1ba14c5b5c0a0befb27906bc00691642d7bdbdce6/pyppmd-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c5cd428715413fe55abf79dc9fc54924ba7e518053e1fc0cbdf80d0d99cf1442", size = 75756 }, + { url = "https://files.pythonhosted.org/packages/85/4a/a7c172cd431c4e1ddf9be349dc4bcfea81c2a236d2fe51bbfdcd697af55a/pyppmd-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0e96cc43f44b7658be2ea764e7fa99c94cb89164dbb7cdf209178effc2168319", size = 47347 }, + { url = "https://files.pythonhosted.org/packages/0d/32/f7357e0412e977ede4d63ba8bf55d014e5ea5b311818b2b0a1fee6d91baa/pyppmd-1.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dd20142869094bceef5ab0b160f4fff790ad1f612313a1e3393a51fc3ba5d57e", size = 46640 }, + { url = "https://files.pythonhosted.org/packages/b5/8e/1f416819f0aab17de47b15b72d0e9b05e2bf795c6e28d9f403ac01398b74/pyppmd-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a4f9b51e45c11e805e74ea6f6355e98a6423b5bbd92f45aceee24761bdc3d3b8", size = 135666 }, + { url = "https://files.pythonhosted.org/packages/73/ac/7d07d3ac6874f235554de392de08e6a369001db43cd6a619af4fbe02fb55/pyppmd-1.1.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:459f85e928fb968d0e34fb6191fd8c4e710012d7d884fa2b317b2e11faac7c59", size = 132892 }, + { url = "https://files.pythonhosted.org/packages/09/76/61db4268a439cfba8736b14130d928d199633fab2360a2c5043332a427d2/pyppmd-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f73cf2aaf60477eef17f5497d14b6099d8be9748390ad2b83d1c88214d050c05", size = 138901 }, + { url = "https://files.pythonhosted.org/packages/8b/9c/546729489ae07c0d7c2bfe37c69ae1cd3ce35a18ab000480ea4e8f12754f/pyppmd-1.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2ea3ae0e92c0b5345cd3a4e145e01bbd79c2d95355481ea5d833b5c0cb202a2d", size = 139725 }, + { url = "https://files.pythonhosted.org/packages/f7/db/4e734e97541554a389e7adb2a2a5c86ad8ae35c4dafe817b12fdc317de1a/pyppmd-1.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:775172c740133c0162a01c1a5443d0e312246881cdd6834421b644d89a634b91", size = 131598 }, + { url = "https://files.pythonhosted.org/packages/b1/8f/530e47290e07d2fdedfd345fc72af08226ccdd4cc913c2b895a8396c17b6/pyppmd-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:14421030f1d46f69829698bdd960698a3b3df0925e3c470e82cfcdd4446b7bc1", size = 142767 }, + { url = "https://files.pythonhosted.org/packages/a5/f9/16e0adfef500b171a96ed3c95f4a4d999f99cc79de3e415146808b19c2fb/pyppmd-1.1.0-cp310-cp310-win32.whl", hash = "sha256:b691264f9962532aca3bba5be848b6370e596d0a2ca722c86df388be08d0568a", size = 41283 }, + { url = "https://files.pythonhosted.org/packages/37/8d/c4846ab632e13ead87189f31bcc51fc825c75078d162a4a9dc8aed0a5b97/pyppmd-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:216b0d969a3f06e35fbfef979706d987d105fcb1e37b0b1324f01ee143719c4a", size = 46078 }, + { url = "https://files.pythonhosted.org/packages/27/0e/9db5d7c6ca3159aa0f07c0f1d5c59079176e7c57740a61aca62a39661178/pyppmd-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1f8c51044ee4df1b004b10bf6b3c92f95ea86cfe1111210d303dca44a56e4282", size = 75781 }, + { url = "https://files.pythonhosted.org/packages/f0/1b/4894b5c71feee76d3dfccf4383b59841f9bfd27aecf912b6542a2ab1e073/pyppmd-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ac25b3a13d1ac9b8f0bde46952e10848adc79d932f2b548a6491ef8825ae0045", size = 47370 }, + { url = "https://files.pythonhosted.org/packages/50/98/57b2c281e546f682279bd4a2577045d1f6d527c8fa2151a990b2a9bc48c2/pyppmd-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c8d3003eebe6aabe22ba744a38a146ed58a25633420d5da882b049342b7c8036", size = 46633 }, + { url = "https://files.pythonhosted.org/packages/06/72/b7e37aa69b7a105bcc119bc171437fbcb104aef2568b68ec8ed21a3fcdd1/pyppmd-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c520656bc12100aa6388df27dd7ac738577f38bf43f4a4bea78e1861e579ea5", size = 138233 }, + { url = "https://files.pythonhosted.org/packages/60/73/4f53a3c7730e1cba3f210b35ed6779e0fe302739196f43452664e079c0b5/pyppmd-1.1.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c2a3e807028159a705951f5cb5d005f94caed11d0984e59cc50506de543e22d", size = 135486 }, + { url = "https://files.pythonhosted.org/packages/31/7c/956ebf1f07506bb59e6f13ef068d91f1bec828758d399b455b175b668f6c/pyppmd-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec8a2447e69444703e2b273247bfcd4b540ec601780eff07da16344c62d2993d", size = 141183 }, + { url = "https://files.pythonhosted.org/packages/73/b4/4863499e012c555f4619dbebc5b83d79818e0161d9b6fb8b1e709fb1d6c7/pyppmd-1.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b9e0c8053e69cad6a92a0889b3324f567afc75475b4f54727de553ac4fc85780", size = 141752 }, + { url = "https://files.pythonhosted.org/packages/b4/cc/44e175222b31f86d0192d1d0d2c46c4bf0e933c9a06a65ff39596ad05666/pyppmd-1.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:5938d256e8d2a2853dc3af8bb58ae6b4a775c46fc891dbe1826a0b3ceb624031", size = 133921 }, + { url = "https://files.pythonhosted.org/packages/f1/d9/2f2e222d43ab274909e8dcd16d25cd4cc0245a8d59f93f8d6397cd4dc49f/pyppmd-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1ce5822d8bea920856232ccfb3c26b56b28b6846ea1b0eb3d5cb9592a026649e", size = 145191 }, + { url = "https://files.pythonhosted.org/packages/6d/e7/1214571442624e2314ed1ed5ba0081358335fc760fb455c3d8df83b118c6/pyppmd-1.1.0-cp311-cp311-win32.whl", hash = "sha256:2a9e894750f2a52b03e3bc0d7cf004d96c3475a59b1af7e797d808d7d29c9ffe", size = 41286 }, + { url = "https://files.pythonhosted.org/packages/8e/7f/d3cc8443bd2b56bc54ea205dcf73d70ef8d4342096ff33fc8719956f45e9/pyppmd-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:969555c72e72fe2b4dd944127521a8f2211caddb5df452bbc2506b5adfac539e", size = 46087 }, + { url = "https://files.pythonhosted.org/packages/bf/0b/4c8e3a92c4366a9aa2d801ab4bd7ba72bd1d214da890dd91ab4d73e52878/pyppmd-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9d6ef8fd818884e914bc209f7961c9400a4da50d178bba25efcef89f09ec9169", size = 76116 }, + { url = "https://files.pythonhosted.org/packages/e1/0b/45fdf5a28c810ed4d3c0cb05ae5346e2972cdbfe89f374b263e07c5b820d/pyppmd-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:95f28e2ecf3a9656bd7e766aaa1162b6872b575627f18715f8b046e8617c124a", size = 47633 }, + { url = "https://files.pythonhosted.org/packages/56/a4/4aa1d36d98f3786c8b12ac96ac8234d7dc3c2a9e8f5174a5698f424099ec/pyppmd-1.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:37f3557ea65ee417abcdf5f49d35df00bb9f6f252639cae57aeefcd0dd596133", size = 46704 }, + { url = "https://files.pythonhosted.org/packages/d9/70/a49389a6666f670db5ecc7caa37030c9a9abfeea455c387172584551a271/pyppmd-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e84b25d088d7727d50218f57f92127cdb839acd6ec3de670b6680a4cf0b2d2a", size = 139145 }, + { url = "https://files.pythonhosted.org/packages/30/4c/f08cdf618744a3cce0da106ecf6e427b24d27b0bb1484afc40b88ca23a39/pyppmd-1.1.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:99ed42891986dac8c2ecf52bddfb777900233d867aa18849dbba6f3335600466", size = 136618 }, + { url = "https://files.pythonhosted.org/packages/bb/e0/afc0fb971c893e9e72cc8d70df93c50b3f3ebb12b4bdb21f869b775faf7e/pyppmd-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6fe69b82634488ada75ba07efb90cd5866fa3d64a2c12932b6e8ae207a14e5f", size = 142757 }, + { url = "https://files.pythonhosted.org/packages/26/b2/793e92c7a66de0b0b8d777c3c4df3ee5a5bec7fbaf0b69ab7374cefefa43/pyppmd-1.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:60981ffde1fe6ade750b690b35318c41a1160a8505597fda2c39a74409671217", size = 142749 }, + { url = "https://files.pythonhosted.org/packages/5e/6e/a1bf750bc7ed025a06600c65917d02e3c6dea7dfa728746c7251d4910d37/pyppmd-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:46e8240315476f57aac23d71e6de003e122b65feba7c68f4cc46a089a82a7cd4", size = 135033 }, + { url = "https://files.pythonhosted.org/packages/1e/ee/4a12a4b1990f1fabb77f9ef94d2cd6c795690eec79ad135b8236dc59dbd2/pyppmd-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c0308e2e76ecb4c878a18c2d7a7c61dbca89b4ef138f65d5f5ead139154dcdea", size = 146510 }, + { url = "https://files.pythonhosted.org/packages/04/cd/a6571420345315f5340ac10897726303ae07260cb025dc4a60371d1e8b97/pyppmd-1.1.0-cp312-cp312-win32.whl", hash = "sha256:b4fa4c27dc1314d019d921f2aa19e17f99250557e7569eeb70e180558f46af74", size = 41332 }, + { url = "https://files.pythonhosted.org/packages/c0/a4/af77129d671d6adcc6c82e1b0f03f0ad0b70c44ac70ed4c72b5c8952553b/pyppmd-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:c269d21e15f4175df27cf00296476097af76941f948734c642d7fb6e85b9b3b9", size = 46193 }, + { url = "https://files.pythonhosted.org/packages/b4/10/144f811290a36d6cf5b5c8ae9b68533abe4bea160285be73435d55c361e1/pyppmd-1.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2aeea1bf585c6b8771fa43a6abd704da92f8a46a6d0020953af15d7f3c82e48c", size = 75760 }, + { url = "https://files.pythonhosted.org/packages/dc/2b/242b6ba7938e77f14dadc0e5d638288b5588f0aff5e5d2c0428726606e5e/pyppmd-1.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7759bdb137694d4ab0cfa5ff2c75c212d90714c7da93544694f68001a0c38e12", size = 47339 }, + { url = "https://files.pythonhosted.org/packages/ec/80/91a13a5d0da916e7243f66839941976b2729bac1d3ca1737c20f1d59b216/pyppmd-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:db64a4fe956a2e700a737a1d019f526e6ccece217c163b28b354a43464cc495b", size = 46636 }, + { url = "https://files.pythonhosted.org/packages/aa/18/f6126af21186eee49f9aa090c36acc9d2bccef4c7d077d23b2f24dfb804c/pyppmd-1.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f788ae8f5a9e79cd777b7969d3401b2a2b87f47abe306c2a03baca30595e9bd", size = 135484 }, + { url = "https://files.pythonhosted.org/packages/03/55/ebfeb5d1085f8a2bd03f498aa7e6ef9635380bf1f88badd1f3b944198ada/pyppmd-1.1.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:324a178935c140210fca2043c688b77e79281da8172d2379a06e094f41735851", size = 132741 }, + { url = "https://files.pythonhosted.org/packages/df/92/f0a7a6e372c4bd659b5528ff179676522aa72bd8c7a071e757a490ff988e/pyppmd-1.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:363030bbcb7902fb9eeb59ffc262581ca5dd7790ba950328242fd2491c54d99b", size = 138761 }, + { url = "https://files.pythonhosted.org/packages/43/32/0f32a70ef3fbe287dc53fd86408e9c2f60515ab356bd728bde9fcebcb598/pyppmd-1.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:31b882584f86440b0ff7906385c9f9d9853e5799197abaafdae2245f87d03f01", size = 139568 }, + { url = "https://files.pythonhosted.org/packages/fd/02/730882ea61653af4ef22b3621e9a03e85e61c90884fb47df4495706439f5/pyppmd-1.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b991b4501492ec3380b605fe30bee0b61480d305e98519d81c2a658b2de01593", size = 131441 }, + { url = "https://files.pythonhosted.org/packages/9e/cd/624b6582766b8e60c4356ebcfe73504c81d1395f36d0d409ff081be56e3d/pyppmd-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b6108044d943b826f97a9e79201242f61392d6c1fadba463b2069c4e6bc961e1", size = 142558 }, + { url = "https://files.pythonhosted.org/packages/65/46/1908b1ef3ba6f1450bc1a3e45a6b39cfe4b33456a312d1add0041ba1bbe4/pyppmd-1.1.0-cp39-cp39-win32.whl", hash = "sha256:c45ce2968b7762d2cacf622b0a8f260295c6444e0883fd21a21017e3eaef16ed", size = 41284 }, + { url = "https://files.pythonhosted.org/packages/63/29/081b03f989deb7ce2f70461dac12ecc422e9abef6b7b7a1933945c96b06f/pyppmd-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:f5289f32ab4ec5f96a95da51309abd1769f928b0bff62047b3bc25c878c16ccb", size = 46071 }, + { url = "https://files.pythonhosted.org/packages/6a/e2/1d5fbd6dde1234b635000072c8d1d87c7ed3acf01a3c4aa8082504d58bc5/pyppmd-1.1.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ad5da9f7592158e6b6b51d7cd15e536d8b23afbb4d22cba4e5744c7e0a3548b1", size = 41505 }, + { url = "https://files.pythonhosted.org/packages/24/66/9215c5dda61b3aa3259902a586dacd198b4b0793ab99228734091b5e7fa7/pyppmd-1.1.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc6543e7d12ef0a1466d291d655e3d6bca59c7336dbb53b62ccdd407822fb52b", size = 44814 }, + { url = "https://files.pythonhosted.org/packages/1a/87/cc2aa429688f238ae30f26b8334194a21e25643d3257c9e5b14cccdc578e/pyppmd-1.1.0-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a5e4008a45910e3c8c227f6f240de67eb14454c015dc3d8060fc41e230f395d3", size = 43629 }, + { url = "https://files.pythonhosted.org/packages/9f/96/cd3f64f6bdce091ffb6d2c1c23dc91e8b94e312a5d08cd648625555fb69e/pyppmd-1.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9301fa39d1fb0ed09a10b4c5d7f0074113e96a1ead16ba7310bedf95f7ef660c", size = 43911 }, + { url = "https://files.pythonhosted.org/packages/e6/ab/02ab90e2dddf2dd55e30e64fa0509627c6e0c86b26503a6df95ae55b1e45/pyppmd-1.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:59521a3c6028da0cb5780ba16880047b00163432a6b975da2f6123adfc1b0be8", size = 42427 }, + { url = "https://files.pythonhosted.org/packages/fa/0e/05db05c0da6a9bbb1f32de107b9f92e95ca9cb407c2082c7a0bee0a8868b/pyppmd-1.1.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:cce8cd2d4ceebe2dbf41db6dfebe4c2e621314b3af8a2df2cba5eb5fa277f122", size = 41500 }, + { url = "https://files.pythonhosted.org/packages/ba/89/48b01bcab274c59548897de336b807777b8e4abbd0465ed37c04152d13e2/pyppmd-1.1.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62e57927dbcb91fb6290a41cd83743b91b9d85858efb16a0dd34fac208ee1c6b", size = 44810 }, + { url = "https://files.pythonhosted.org/packages/a9/e4/bf4aa2305eda1bdf530d8127e07dd324ae923e664db3aa7bf153f1d749ec/pyppmd-1.1.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:435317949a6f35e54cdf08e0af6916ace427351e7664ac1593980114668f0aaa", size = 43625 }, + { url = "https://files.pythonhosted.org/packages/fc/be/33478c13fad90049c35034d9f7de658353f82fc2cd91c57efd904b71ebb7/pyppmd-1.1.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f66b0d0e32b8fb8707f1d2552f13edfc2917e8ed0bdf4d62e2ce190d2c70834", size = 43909 }, + { url = "https://files.pythonhosted.org/packages/60/34/922d8ca6879f08d17f9771e6ef65e9491c2ebbd48934997a3ff01285e55e/pyppmd-1.1.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:650a663a591e06fb8096c213f4070b158981c8c3bf9c166ce7e4c360873f2750", size = 42422 }, +] + [[package]] name = "pyright" version = "1.1.373" @@ -2073,6 +2396,96 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ca/63/27e6142b4f67a442ee480986ca5b88edb01462dd2319843057683a5148bd/pyzmq-26.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:4f78c88905461a9203eac9faac157a2a0dbba84a0fd09fd29315db27be40af9f", size = 550757 }, ] +[[package]] +name = "pyzstd" +version = "0.16.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/62/14/878fee4072cecb1cc6e061c7d0d933e481389c27de939538c9cc3f18894a/pyzstd-0.16.2.tar.gz", hash = "sha256:179c1a2ea1565abf09c5f2fd72f9ce7c54b2764cf7369e05c0bfd8f1f67f63d2", size = 789505 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/a9/efad061c5a982f859ba8bf5de565d73567f87ad8bba3364fe28e9a8672b6/pyzstd-0.16.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:637376c8f8cbd0afe1cab613f8c75fd502bd1016bf79d10760a2d5a00905fe62", size = 372191 }, + { url = "https://files.pythonhosted.org/packages/b6/36/eb6dcfacb273ca13dfa20d296f27ffd0a6c53677965f868625edf764b71e/pyzstd-0.16.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3e7a7118cbcfa90ca2ddbf9890c7cb582052a9a8cf2b7e2c1bbaf544bee0f16a", size = 295083 }, + { url = "https://files.pythonhosted.org/packages/fb/76/a7862487402123f221439808ed50915e00cfc8e1df7365af366610176347/pyzstd-0.16.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a74cb1ba05876179525144511eed3bd5a509b0ab2b10632c1215a85db0834dfd", size = 390166 }, + { url = "https://files.pythonhosted.org/packages/b8/52/1e1ab63026d67f18b9841285576d59bb799b838a5de4f852ad9e054674a1/pyzstd-0.16.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c084dde218ffbf112e507e72cbf626b8f58ce9eb23eec129809e31037984662", size = 472043 }, + { url = "https://files.pythonhosted.org/packages/0d/24/14c8948b9d16d399ff80504bc404bb091b0eb5339f6fbdad0481da751c09/pyzstd-0.16.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4646459ebd3d7a59ddbe9312f020bcf7cdd1f059a2ea07051258f7af87a0b31", size = 415258 }, + { url = "https://files.pythonhosted.org/packages/6b/3e/e4c7f449af9d19975ff5d333a58330317cf8b05fe4754106c694a29e7c25/pyzstd-0.16.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14bfc2833cc16d7657fc93259edeeaa793286e5031b86ca5dc861ba49b435fce", size = 413680 }, + { url = "https://files.pythonhosted.org/packages/10/09/8918853028cf593c141456b9a42d68420beec3f16a8cc4f1aa5d0b8b0c84/pyzstd-0.16.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f27d488f19e5bf27d1e8aa1ae72c6c0a910f1e1ffbdf3c763d02ab781295dd27", size = 412630 }, + { url = "https://files.pythonhosted.org/packages/47/20/5a4c899530571e0e8ecdcb9dc7e3fc38491d4b342fbd7d8413805c88013b/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:91e134ca968ff7dcfa8b7d433318f01d309b74ee87e0d2bcadc117c08e1c80db", size = 404980 }, + { url = "https://files.pythonhosted.org/packages/0a/1d/aeeeebb702d3500a01b5b1029ba1716aea3afa75e8aacb904806b3f1afe5/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:6b5f64cd3963c58b8f886eb6139bb8d164b42a74f8a1bb95d49b4804f4592d61", size = 418000 }, + { url = "https://files.pythonhosted.org/packages/fc/0c/66ca36d24ad97af40a8fe8de9e3f316a5f4fd2fb3cab8634a2f7da5571c8/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:0b4a8266871b9e0407f9fd8e8d077c3558cf124d174e6357b523d14f76971009", size = 485576 }, + { url = "https://files.pythonhosted.org/packages/39/66/6c1de1347de94aa85f60e854cccae0948bda2eda2351e4d47c8bb0a7cf18/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:1bb19f7acac30727354c25125922aa59f44d82e0e6a751df17d0d93ff6a73853", size = 564542 }, + { url = "https://files.pythonhosted.org/packages/6d/46/75365a3ab279d58e69d410ce0a21527e689fa651837227e23dee294d096f/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3008325b7368e794d66d4d98f2ee1d867ef5afd09fd388646ae02b25343c420d", size = 430619 }, + { url = "https://files.pythonhosted.org/packages/0d/62/17bf81d42acbd39bffdea559b6fbd7ec331cd74bc52f249e536fefe5480d/pyzstd-0.16.2-cp310-cp310-win32.whl", hash = "sha256:66f2d5c0bbf5bf32c577aa006197b3525b80b59804450e2c32fbcc2d16e850fd", size = 218224 }, + { url = "https://files.pythonhosted.org/packages/f7/b6/281245890df08a567186c6e262c43d68581291cca107c8d7304c37708e46/pyzstd-0.16.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fe5f5459ebe1161095baa7a86d04ab625b35148f6c425df0347ed6c90a2fd58", size = 245012 }, + { url = "https://files.pythonhosted.org/packages/10/5a/19d7aec81853f6dc53eabad388227e3beecfaca4788af23b8807a0ea2112/pyzstd-0.16.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1bdbe7f01c7f37d5cd07be70e32a84010d7dfd6677920c0de04cf7d245b60d", size = 372192 }, + { url = "https://files.pythonhosted.org/packages/29/35/2eb025e6a0fff49b5de8bea20e82e4d7d5456e634bf3809123fbe5e5f194/pyzstd-0.16.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1882a3ceaaf9adc12212d587d150ec5e58cfa9a765463d803d739abbd3ac0f7a", size = 295084 }, + { url = "https://files.pythonhosted.org/packages/04/1f/03785d7ff1ce73b9347533f798cb27afa57768e66012f97b18b7b7303158/pyzstd-0.16.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea46a8b9d60f6a6eba29facba54c0f0d70328586f7ef0da6f57edf7e43db0303", size = 390167 }, + { url = "https://files.pythonhosted.org/packages/b7/59/e307622115a2df30075efbd28933dc0ad8f2007c5ba5a3eb49c956de3d56/pyzstd-0.16.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d7865bc06589cdcecdede0deefe3da07809d5b7ad9044c224d7b2a0867256957", size = 472038 }, + { url = "https://files.pythonhosted.org/packages/97/21/870fda5454240089e9c37625320580d392b03beaeae4889c67c0a21c4d34/pyzstd-0.16.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:52f938a65b409c02eb825e8c77fc5ea54508b8fc44b5ce226db03011691ae8cc", size = 415217 }, + { url = "https://files.pythonhosted.org/packages/3c/35/b33faeeb9c96fddd08bf7871c9f5c4638c32ad79227155922fd4a63190c5/pyzstd-0.16.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e97620d3f53a0282947304189deef7ca7f7d0d6dfe15033469dc1c33e779d5e5", size = 413714 }, + { url = "https://files.pythonhosted.org/packages/aa/a3/b9058dd43eb52025a2ca78946dcb9ef9d8984acac172a698bcf12712217c/pyzstd-0.16.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7c40e9983d017108670dc8df68ceef14c7c1cf2d19239213274783041d0e64c", size = 412568 }, + { url = "https://files.pythonhosted.org/packages/12/31/fe7d462c912f2040775bfa2af4327f9fcebb16e8fa9c3bfa058bc1306722/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7cd4b3b2c6161066e4bde6af1cf78ed3acf5d731884dd13fdf31f1db10830080", size = 404988 }, + { url = "https://files.pythonhosted.org/packages/48/4c/582aca0e5210436499bce1639a8d15da3f76f8d5827da1aa3eeb2c4e271c/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:454f31fd84175bb203c8c424f2255a343fa9bd103461a38d1bf50487c3b89508", size = 417961 }, + { url = "https://files.pythonhosted.org/packages/39/e9/54f53641ff10b4ea18d3ba159b03bd07e6ae5a5b7ae01f1329b0c35b8ca2/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:5ef754a93743f08fb0386ce3596780bfba829311b49c8f4107af1a4bcc16935d", size = 485587 }, + { url = "https://files.pythonhosted.org/packages/ce/65/25243b3fea9e52a20bfece1b12e3d3ee3125f17b1735aab08cb9a7a760b4/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:be81081db9166e10846934f0e3576a263cbe18d81eca06e6a5c23533f8ce0dc6", size = 564543 }, + { url = "https://files.pythonhosted.org/packages/3b/3c/324b8ddca55b4b073b413cea3e0587af3c8153ccf7d6d63ed294831f2095/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:738bcb2fa1e5f1868986f5030955e64de53157fa1141d01f3a4daf07a1aaf644", size = 430628 }, + { url = "https://files.pythonhosted.org/packages/db/a1/aca18925e23bceb833fc742ebaf87aa9d1ba8b178f0332bd108fc8966482/pyzstd-0.16.2-cp311-cp311-win32.whl", hash = "sha256:0ea214c9b97046867d1657d55979021028d583704b30c481a9c165191b08d707", size = 218215 }, + { url = "https://files.pythonhosted.org/packages/c0/7f/0f5d1d1891e6c6e14d846d2881a06ab7e5e97cabeb5e1e9e53debec4091a/pyzstd-0.16.2-cp311-cp311-win_amd64.whl", hash = "sha256:c17c0fc02f0e75b0c7cd21f8eaf4c6ce4112333b447d93da1773a5f705b2c178", size = 245055 }, + { url = "https://files.pythonhosted.org/packages/28/15/20046759d138733e7150afa6aa15f322022d7587968e2dbd5b36fbf8aa86/pyzstd-0.16.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d4081fd841a9efe9ded7290ee7502dbf042c4158b90edfadea3b8a072c8ec4e1", size = 373230 }, + { url = "https://files.pythonhosted.org/packages/51/8d/55b536edaecf19d2f8dbd8fbaefd184f2f9cc6b71d241caa6d86bed96813/pyzstd-0.16.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fd3fa45d2aeb65367dd702806b2e779d13f1a3fa2d13d5ec777cfd09de6822de", size = 295699 }, + { url = "https://files.pythonhosted.org/packages/11/14/086e7f690154c6f3d9bdb46da26a4cd3c9e0b284346ce10943711ca48c32/pyzstd-0.16.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8b5f0d2c07994a5180d8259d51df6227a57098774bb0618423d7eb4a7303467", size = 390556 }, + { url = "https://files.pythonhosted.org/packages/90/d2/c6d854705d6fa0ad876209b4ba796ab31d85b710d1459029f2cb41085a8d/pyzstd-0.16.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60c9d25b15c7ae06ed5d516d096a0d8254f9bed4368b370a09cccf191eaab5cb", size = 472928 }, + { url = "https://files.pythonhosted.org/packages/aa/38/f97dd871e446adc834349caa605dbaf5bac86763a255f62c809cc2459c85/pyzstd-0.16.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:29acf31ce37254f6cad08deb24b9d9ba954f426fa08f8fae4ab4fdc51a03f4ae", size = 416057 }, + { url = "https://files.pythonhosted.org/packages/53/be/0c5ad7bf29dc890f6a3303760b9802aeeafa4e3ffb598de625f501986bfe/pyzstd-0.16.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ec77612a17697a9f7cf6634ffcee616eba9b997712fdd896e77fd19ab3a0618", size = 414613 }, + { url = "https://files.pythonhosted.org/packages/1f/1a/d3a1edcd59e2f62a35ac6257d2b86a2c872ae9a8e925380620a8db0d9a9a/pyzstd-0.16.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:313ea4974be93be12c9a640ab40f0fc50a023178aae004a8901507b74f190173", size = 413236 }, + { url = "https://files.pythonhosted.org/packages/f2/8d/912430c2310466c14a89a5a529b72eddef7e73fa733806dbe0b030cf3495/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e91acdefc8c2c6c3b8d5b1b5fe837dce4e591ecb7c0a2a50186f552e57d11203", size = 405536 }, + { url = "https://files.pythonhosted.org/packages/9e/83/4edb419a13b9d1e1debc01e88084eba93a5f7c10ef198da11f6782857c73/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:929bd91a403539e72b5b5cb97f725ac4acafe692ccf52f075e20cd9bf6e5493d", size = 419145 }, + { url = "https://files.pythonhosted.org/packages/8f/e9/62a169eddc37aefac480ee3b3318c221f6731e1e342dafd9e05b7fdaa7c5/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:740837a379aa32d110911ebcbbc524f9a9b145355737527543a884bd8777ca4f", size = 487157 }, + { url = "https://files.pythonhosted.org/packages/57/9d/5949f2a0144d1f99fab7914f854b582d2784c73139cc190e603e4d6b7b37/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:adfc0e80dd157e6d1e0b0112c8ecc4b58a7a23760bd9623d74122ef637cfbdb6", size = 565918 }, + { url = "https://files.pythonhosted.org/packages/de/ce/647b9c7602ac477c9e62cf9399810f72bb5dba8f508e7cdf8be1d260e6f9/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:79b183beae1c080ad3dca39019e49b7785391947f9aab68893ad85d27828c6e7", size = 431373 }, + { url = "https://files.pythonhosted.org/packages/8b/fb/4141e3d4549eea26e5a59ec723eade271980816cb2ed7613df855baa672f/pyzstd-0.16.2-cp312-cp312-win32.whl", hash = "sha256:b8d00631a3c466bc313847fab2a01f6b73b3165de0886fb03210e08567ae3a89", size = 218541 }, + { url = "https://files.pythonhosted.org/packages/51/b9/e1373b179129c2095d70bd1df02a51d388f4c7e4ecb62acb4e5e9570269b/pyzstd-0.16.2-cp312-cp312-win_amd64.whl", hash = "sha256:c0d43764e9a60607f35d8cb3e60df772a678935ab0e02e2804d4147377f4942c", size = 245320 }, + { url = "https://files.pythonhosted.org/packages/66/10/cc7c764c7673f1af1728abdcf58e58f88ef5d44ab4500677a2b7b4c01e7d/pyzstd-0.16.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3ae9ae7ad730562810912d7ecaf1fff5eaf4c726f4b4dfe04784ed5f06d7b91f", size = 373223 }, + { url = "https://files.pythonhosted.org/packages/3f/a7/bcaf7d635ee929dd4d08ae1c35101892db56a11542471eecfbf46b9dd988/pyzstd-0.16.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2ce8d3c213f76a564420f3d0137066ac007ce9fb4e156b989835caef12b367a7", size = 295701 }, + { url = "https://files.pythonhosted.org/packages/93/49/a604113a2f3135b29371a894c0faad22d7ea3f7b58f38d77baad8a817483/pyzstd-0.16.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c2c14dac23c865e2d78cebd9087e148674b7154f633afd4709b4cd1520b99a61", size = 392395 }, + { url = "https://files.pythonhosted.org/packages/b0/38/886ecf3ebb13a4b6e3ee85f448f54eef37a5ae2b453bd9d5d9edc909e119/pyzstd-0.16.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4527969d66a943e36ef374eda847e918077de032d58b5df84d98ffd717b6fa77", size = 474523 }, + { url = "https://files.pythonhosted.org/packages/14/98/121da6ac072c00090c218b4888ef00ead15979f09a657d9a5ff770d6bb17/pyzstd-0.16.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd8256149b88e657e99f31e6d4b114c8ff2935951f1d8bb8e1fe501b224999c0", size = 417974 }, + { url = "https://files.pythonhosted.org/packages/b6/ba/56652a67c0bcfaceb2945e5f07d5aa21af86e07cf33d1ae47bb3529a56c3/pyzstd-0.16.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5bd1f1822d65c9054bf36d35307bf8ed4aa2d2d6827431761a813628ff671b1d", size = 414587 }, + { url = "https://files.pythonhosted.org/packages/cc/30/cab6f45101f0113ced609ef65482aedd276e0f022d9f25a327d4284142f5/pyzstd-0.16.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6733f4d373ec9ad2c1976cf06f973a3324c1f9abe236d114d6bb91165a397d", size = 415071 }, + { url = "https://files.pythonhosted.org/packages/6d/44/2187fc8a46662926943aeb16d639dd4f3d06267c7e8abb2c6f97700ab11c/pyzstd-0.16.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7bec165ab6524663f00b69bfefd13a46a69fed3015754abaf81b103ec73d92c6", size = 407835 }, + { url = "https://files.pythonhosted.org/packages/de/d5/6edca97d5453cba820d2ad5630e6ec1fcfad66f69af5ad7d6c688ea301be/pyzstd-0.16.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e4460fa6949aac6528a1ad0de8871079600b12b3ef4db49316306786a3598321", size = 421755 }, + { url = "https://files.pythonhosted.org/packages/54/c1/1a0339e014ed97f4e6fd9166b0409ceda8f32e28e8ecda70fd7bb0915566/pyzstd-0.16.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:75df79ea0315c97d88337953a17daa44023dbf6389f8151903d371513f503e3c", size = 489174 }, + { url = "https://files.pythonhosted.org/packages/07/01/c65f2c9f0b902b33efcb0bdf3cbd07fc828fda6ff6333189eb71cf7acc60/pyzstd-0.16.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:93e1d45f4a196afb6f18682c79bdd5399277ead105b67f30b35c04c207966071", size = 573025 }, + { url = "https://files.pythonhosted.org/packages/a7/54/7ab9cc54171b7f8bb97cfd1c1aa7fcb706a4babeb629732529d8111bc4e6/pyzstd-0.16.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:075e18b871f38a503b5d23e40a661adfc750bd4bd0bb8b208c1e290f3ceb8fa2", size = 429582 }, + { url = "https://files.pythonhosted.org/packages/6c/a5/f9c950bb378dd1335bc4cc56444ec2ab40b1dab085c5798c5d16a9bf9d0b/pyzstd-0.16.2-cp313-cp313-win32.whl", hash = "sha256:9e4295eb299f8d87e3487852bca033d30332033272a801ca8130e934475e07a9", size = 218544 }, + { url = "https://files.pythonhosted.org/packages/9a/df/a15b9a8a59cd9908ae2b70bce2cb4ac3e2d7da11414ee0d0ceb46e4d0439/pyzstd-0.16.2-cp313-cp313-win_amd64.whl", hash = "sha256:18deedc70f858f4cf574e59f305d2a0678e54db2751a33dba9f481f91bc71c28", size = 245313 }, + { url = "https://files.pythonhosted.org/packages/e0/38/43002103a545bc953e532973596e905550e9626973c1b282e04e01038ac6/pyzstd-0.16.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a9892b707ef52f599098b1e9528df0e7849c5ec01d3e8035fb0e67de4b464839", size = 372192 }, + { url = "https://files.pythonhosted.org/packages/61/be/28dfeba9dbad8ed19d6aefa0d6623d1ee97e83c6c1e97910439428655f28/pyzstd-0.16.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4fbd647864341f3c174c4a6d7f20e6ea6b4be9d840fb900dc0faf0849561badc", size = 295080 }, + { url = "https://files.pythonhosted.org/packages/63/c2/c7e5244f2dde72df3fb2b7b952e8d01bac20cd78dc0d585d0a060ca565b0/pyzstd-0.16.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20ac2c15656cc6194c4fed1cb0e8159f9394d4ea1d58be755448743d2ec6c9c4", size = 390165 }, + { url = "https://files.pythonhosted.org/packages/ff/30/52560cb88179fa3ff7536429c0d7b83aeecea86ecb2d180a4afc991502e5/pyzstd-0.16.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b239fb9a20c1be3374b9a2bd183ba624fd22ad7a3f67738c0d80cda68b4ae1d3", size = 472040 }, + { url = "https://files.pythonhosted.org/packages/69/a7/ab1e19626da5a8ff58493d6928d9d0da4931034e7a124949bf1a1705daaf/pyzstd-0.16.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cc52400412cdae2635e0978b8d6bcc0028cc638fdab2fd301f6d157675d26896", size = 415255 }, + { url = "https://files.pythonhosted.org/packages/28/0d/bf7c9388fe43c7051a2ced4645e58a493a35c62e68307b5aaf0fb129b008/pyzstd-0.16.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b766a6aeb8dbb6c46e622e7a1aebfa9ab03838528273796941005a5ce7257b1", size = 413679 }, + { url = "https://files.pythonhosted.org/packages/58/2a/1e0738740a8bd2b1f4a74be86297c5776936b66b3a5340d8e4ae84c5844f/pyzstd-0.16.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd4b8676052f9d59579242bf3cfe5fd02532b6a9a93ab7737c118ae3b8509dc", size = 412623 }, + { url = "https://files.pythonhosted.org/packages/23/d5/7cbfbebbb3ffccb0626fc2fab622fb5a10cf66c2c60481f51e46a92eb2c5/pyzstd-0.16.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1c6c0a677aac7c0e3d2d2605d4d68ffa9893fdeeb2e071040eb7c8750969d463", size = 404981 }, + { url = "https://files.pythonhosted.org/packages/a7/b0/6ac198c753cc135357630e856f40f5998c2d28609713ae2830c679e8248c/pyzstd-0.16.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:15f9c2d612e7e2023d68d321d1b479846751f792af89141931d44e82ae391394", size = 417997 }, + { url = "https://files.pythonhosted.org/packages/c6/8f/0e5685efbf24ae62e135549e37947ca7919616b81108584112e25dd1a55a/pyzstd-0.16.2-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:11740bff847aad23beef4085a1bb767d101895881fe891f0a911aa27d43c372c", size = 485576 }, + { url = "https://files.pythonhosted.org/packages/30/d6/bf2f05752082967ac748d7c2d7c5a71097ac6fc1b902b5d34764cd0c12f7/pyzstd-0.16.2-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:b9067483ebe860e4130a03ee665b3d7be4ec1608b208e645d5e7eb3492379464", size = 564538 }, + { url = "https://files.pythonhosted.org/packages/d8/97/1081cc3cbf5eeb6cf4e385226e9989fdebb61f8e48baa210eb774145e667/pyzstd-0.16.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:988f0ba19b14c2fe0afefc444ac1edfb2f497b7d7c3212b2f587504cc2ec804e", size = 430615 }, + { url = "https://files.pythonhosted.org/packages/e0/a7/2a82fbb248b951434306dd77e969fb99305968904c9a7494574d696b1392/pyzstd-0.16.2-cp39-cp39-win32.whl", hash = "sha256:8855acb1c3e3829030b9e9e9973b19e2d70f33efb14ad5c474b4d086864c959c", size = 218215 }, + { url = "https://files.pythonhosted.org/packages/9d/bf/e529ff84b87c8f978ab35906921ac54841270562e65bcb5d0dd9d3240204/pyzstd-0.16.2-cp39-cp39-win_amd64.whl", hash = "sha256:018e88378df5e76f5e1d8cf4416576603b6bc4a103cbc66bb593eaac54c758de", size = 245047 }, + { url = "https://files.pythonhosted.org/packages/f9/ad/c09fb722c12a82b826c97efc50a919e229bfbaf644f5a140adcd71941473/pyzstd-0.16.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4b631117b97a42ff6dfd0ffc885a92fff462d7c34766b28383c57b996f863338", size = 364187 }, + { url = "https://files.pythonhosted.org/packages/57/f9/93175fe72f85fb675fe04abca296fe583112a25d0ec7faa026288d9463c2/pyzstd-0.16.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:56493a3fbe1b651a02102dd0902b0aa2377a732ff3544fb6fb3f114ca18db52f", size = 279825 }, + { url = "https://files.pythonhosted.org/packages/8a/de/0b40acf76d7ed1f7975877535e004de85ec2e869632754b5d4d389258b8a/pyzstd-0.16.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1eae9bdba4a1e5d3181331f403114ff5b8ce0f4b569f48eba2b9beb2deef1e4", size = 321313 }, + { url = "https://files.pythonhosted.org/packages/41/5e/00102bacd1a7c957c88098f3ae2cdac17842ac0f94d2e685ff5b75a05730/pyzstd-0.16.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1be6972391c8aeecc7e61feb96ffc8e77a401bcba6ed994e7171330c45a1948", size = 344376 }, + { url = "https://files.pythonhosted.org/packages/a3/95/27a7da3dbd4460cd9432bdc22d9d5f8ec77c86275d069020fa74ea280f7f/pyzstd-0.16.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:761439d687e3a5687c2ff5c6a1190e1601362a4a3e8c6c82ff89719d51d73e19", size = 328591 }, + { url = "https://files.pythonhosted.org/packages/c2/03/8f4d5fd45f6bfad66d67cdf583492a9f52a21049f60e6b36a7e9f8aa7adc/pyzstd-0.16.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f5fbdb8cf31b60b2dc586fecb9b73e2f172c21a0b320ed275f7b8d8a866d9003", size = 240786 }, + { url = "https://files.pythonhosted.org/packages/91/f6/bd63e2587e0ec40abd9f92278a442bc28b7ff109e418d1240ee2eb6536aa/pyzstd-0.16.2-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:183f26e34f9becf0f2db38be9c0bfb136753d228bcb47c06c69175901bea7776", size = 364180 }, + { url = "https://files.pythonhosted.org/packages/ac/13/d4c68ad926e79d734f57b26d49447908e8dab7f5c066d3a013b0d0cfa2be/pyzstd-0.16.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:88318b64b5205a67748148d6d244097fa6cf61fcea02ad3435511b9e7155ae16", size = 279816 }, + { url = "https://files.pythonhosted.org/packages/b2/ba/76f0b75ec9e9fc3914496e036f99f345d5e0a99cb7070341f9becdaba2b8/pyzstd-0.16.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73142aa2571b6480136a1865ebda8257e09eabbc8bcd54b222202f6fa4febe1e", size = 321308 }, + { url = "https://files.pythonhosted.org/packages/a6/ea/9fe52bd777f33f007287f1a37bada7af5cf33d64904360c17bb64fefca21/pyzstd-0.16.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d3f8877c29a97f1b1bba16f3d3ab01ad10ad3da7bad317aecf36aaf8848b37c", size = 344368 }, + { url = "https://files.pythonhosted.org/packages/cc/c0/509077f73fc8e156ceeefb41d4b7e04aceb71b2339084fcd62d0ad3bfd75/pyzstd-0.16.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1f25754562473ac7de856b8331ebd5964f5d85601045627a5f0bb0e4e899990", size = 328585 }, + { url = "https://files.pythonhosted.org/packages/14/74/a854ada61bf4c3c2ad239ec2bd1ff73cc0d718ccbcc56e3ced94e878fd50/pyzstd-0.16.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:6ce17e84310080c55c02827ad9bb17893c00a845c8386a328b346f814aabd2c1", size = 240783 }, +] + [[package]] name = "referencing" version = "0.35.1" @@ -2263,26 +2676,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.1.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/25/4c/2f786388acd82c295eedc4afeede7ef4b29cf27277151d8d13be906bac70/ruff-0.1.6.tar.gz", hash = "sha256:1b09f29b16c6ead5ea6b097ef2764b42372aebe363722f1605ecbcd2b9207184", size = 1719627 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/75/5054ec93ec0d5db26e218cb2814ddaa085ba1f29fad0ec56dd8107a97688/ruff-0.1.6-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:88b8cdf6abf98130991cbc9f6438f35f6e8d41a02622cc5ee130a02a0ed28703", size = 11628468 }, - { url = "https://files.pythonhosted.org/packages/a2/91/8b2920f6026c069ae0802fc3c44f7337e04bf2a198ce94bfab360073477a/ruff-0.1.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5c549ed437680b6105a1299d2cd30e4964211606eeb48a0ff7a93ef70b902248", size = 5990066 }, - { url = "https://files.pythonhosted.org/packages/df/1e/03ef0cc5c7d03e50d4f954218551d6001f1f70e6f391cdb678efb5c6e6ab/ruff-0.1.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cf5f701062e294f2167e66d11b092bba7af6a057668ed618a9253e1e90cfd76", size = 5929313 }, - { url = "https://files.pythonhosted.org/packages/92/7c/38fd1b9cb624f5725a6a08c81bf7e823c64b28622ffcb4369c56dc0a16d0/ruff-0.1.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:05991ee20d4ac4bb78385360c684e4b417edd971030ab12a4fbd075ff535050e", size = 5627943 }, - { url = "https://files.pythonhosted.org/packages/c7/c3/98e3d0eb92e5a2ec10f76c71067640b6f21def23c3b1ff8f08ab6348255e/ruff-0.1.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87455a0c1f739b3c069e2f4c43b66479a54dea0276dd5d4d67b091265f6fd1dc", size = 6074206 }, - { url = "https://files.pythonhosted.org/packages/e8/33/62fb966eb70d9bb45ddf5023d40e26946a5e5127d99956b84c8a9a76b153/ruff-0.1.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:683aa5bdda5a48cb8266fcde8eea2a6af4e5700a392c56ea5fb5f0d4bfdc0240", size = 6743205 }, - { url = "https://files.pythonhosted.org/packages/c7/f1/60d43182f98113156a1b21a17f30541dda9f5ffcfeedc2b54dc030a2c413/ruff-0.1.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:137852105586dcbf80c1717facb6781555c4e99f520c9c827bd414fac67ddfb6", size = 6600581 }, - { url = "https://files.pythonhosted.org/packages/09/92/36850598e84f75cfe8edd252dbf40442b4cc226ed2c76206a9b3cbfb9986/ruff-0.1.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd98138a98d48a1c36c394fd6b84cd943ac92a08278aa8ac8c0fdefcf7138f35", size = 7708098 }, - { url = "https://files.pythonhosted.org/packages/3b/2f/8ef67614631622aa3ea79b27e01ac86d7f90a988520454e3a84cb2fd890f/ruff-0.1.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a0cd909d25f227ac5c36d4e7e681577275fb74ba3b11d288aff7ec47e3ae745", size = 6295145 }, - { url = "https://files.pythonhosted.org/packages/3c/4b/af366db98d15efe83fd3e3aae7319d3897e3475fc53a2f1b0287c8255422/ruff-0.1.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e8fd1c62a47aa88a02707b5dd20c5ff20d035d634aa74826b42a1da77861b5ff", size = 5878111 }, - { url = "https://files.pythonhosted.org/packages/bf/af/25b794e750f1d74a83ce6b16625e3306beeb2161c517b9d883958de05526/ruff-0.1.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fd89b45d374935829134a082617954120d7a1470a9f0ec0e7f3ead983edc48cc", size = 5629339 }, - { url = "https://files.pythonhosted.org/packages/81/b0/92c4cb6bceb19ebd27cedd1f45b337f7fd5397e6b760094831266be59661/ruff-0.1.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:491262006e92f825b145cd1e52948073c56560243b55fb3b4ecb142f6f0e9543", size = 5959772 }, - { url = "https://files.pythonhosted.org/packages/11/02/3a7e3101d88b113f326e0fdf3f566fba2600fc4b1fd828d56027d293e22d/ruff-0.1.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ea284789861b8b5ca9d5443591a92a397ac183d4351882ab52f6296b4fdd5462", size = 6356759 }, - { url = "https://files.pythonhosted.org/packages/fc/93/8a4b3523c4564168aab720d8361b4bf12173b099509caaac93168b72927a/ruff-0.1.6-py3-none-win32.whl", hash = "sha256:1610e14750826dfc207ccbcdd7331b6bd285607d4181df9c1c6ae26646d6848a", size = 5835632 }, - { url = "https://files.pythonhosted.org/packages/40/e3/55ae013087bd892dd61c25895a119ffec2f9b9f7c3d1fd0cea5f5f7bd74a/ruff-0.1.6-py3-none-win_amd64.whl", hash = "sha256:4558b3e178145491e9bc3b2ee3c4b42f19d19384eaa5c59d10acf6e8f8b57e33", size = 6226075 }, - { url = "https://files.pythonhosted.org/packages/21/f7/fdce733e594e1b274fc9232256be6a4e03e8d2cb9e354c783801191e2f4d/ruff-0.1.6-py3-none-win_arm64.whl", hash = "sha256:03910e81df0d8db0e30050725a5802441c2022ea3ae4fe0609b76081731accbc", size = 5965823 }, +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/2b/01245f4f3a727d60bebeacd7ee6d22586c7f62380a2597ddb22c2f45d018/ruff-0.8.2.tar.gz", hash = "sha256:b84f4f414dda8ac7f75075c1fa0b905ac0ff25361f42e6d5da681a465e0f78e5", size = 3349020 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/29/366be70216dba1731a00a41f2f030822b0c96c7c4f3b2c0cdce15cbace74/ruff-0.8.2-py3-none-linux_armv6l.whl", hash = "sha256:c49ab4da37e7c457105aadfd2725e24305ff9bc908487a9bf8d548c6dad8bb3d", size = 10530649 }, + { url = "https://files.pythonhosted.org/packages/63/82/a733956540bb388f00df5a3e6a02467b16c0e529132625fe44ce4c5fb9c7/ruff-0.8.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ec016beb69ac16be416c435828be702ee694c0d722505f9c1f35e1b9c0cc1bf5", size = 10274069 }, + { url = "https://files.pythonhosted.org/packages/3d/12/0b3aa14d1d71546c988a28e1b412981c1b80c8a1072e977a2f30c595cc4a/ruff-0.8.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:f05cdf8d050b30e2ba55c9b09330b51f9f97d36d4673213679b965d25a785f3c", size = 9909400 }, + { url = "https://files.pythonhosted.org/packages/23/08/f9f08cefb7921784c891c4151cce6ed357ff49e84b84978440cffbc87408/ruff-0.8.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:60f578c11feb1d3d257b2fb043ddb47501ab4816e7e221fbb0077f0d5d4e7b6f", size = 10766782 }, + { url = "https://files.pythonhosted.org/packages/e4/71/bf50c321ec179aa420c8ec40adac5ae9cc408d4d37283a485b19a2331ceb/ruff-0.8.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cbd5cf9b0ae8f30eebc7b360171bd50f59ab29d39f06a670b3e4501a36ba5897", size = 10286316 }, + { url = "https://files.pythonhosted.org/packages/f2/83/c82688a2a6117539aea0ce63fdf6c08e60fe0202779361223bcd7f40bd74/ruff-0.8.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b402ddee3d777683de60ff76da801fa7e5e8a71038f57ee53e903afbcefdaa58", size = 11338270 }, + { url = "https://files.pythonhosted.org/packages/7f/d7/bc6a45e5a22e627640388e703160afb1d77c572b1d0fda8b4349f334fc66/ruff-0.8.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:705832cd7d85605cb7858d8a13d75993c8f3ef1397b0831289109e953d833d29", size = 12058579 }, + { url = "https://files.pythonhosted.org/packages/da/3b/64150c93946ec851e6f1707ff586bb460ca671581380c919698d6a9267dc/ruff-0.8.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:32096b41aaf7a5cc095fa45b4167b890e4c8d3fd217603f3634c92a541de7248", size = 11615172 }, + { url = "https://files.pythonhosted.org/packages/e4/9e/cf12b697ea83cfe92ec4509ae414dc4c9b38179cc681a497031f0d0d9a8e/ruff-0.8.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e769083da9439508833cfc7c23e351e1809e67f47c50248250ce1ac52c21fb93", size = 12882398 }, + { url = "https://files.pythonhosted.org/packages/a9/27/96d10863accf76a9c97baceac30b0a52d917eb985a8ac058bd4636aeede0/ruff-0.8.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fe716592ae8a376c2673fdfc1f5c0c193a6d0411f90a496863c99cd9e2ae25d", size = 11176094 }, + { url = "https://files.pythonhosted.org/packages/eb/10/cd2fd77d4a4e7f03c29351be0f53278a393186b540b99df68beb5304fddd/ruff-0.8.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:81c148825277e737493242b44c5388a300584d73d5774defa9245aaef55448b0", size = 10771884 }, + { url = "https://files.pythonhosted.org/packages/71/5d/beabb2ff18870fc4add05fa3a69a4cb1b1d2d6f83f3cf3ae5ab0d52f455d/ruff-0.8.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d261d7850c8367704874847d95febc698a950bf061c9475d4a8b7689adc4f7fa", size = 10382535 }, + { url = "https://files.pythonhosted.org/packages/ae/29/6b3fdf3ad3e35b28d87c25a9ff4c8222ad72485ab783936b2b267250d7a7/ruff-0.8.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1ca4e3a87496dc07d2427b7dd7ffa88a1e597c28dad65ae6433ecb9f2e4f022f", size = 10886995 }, + { url = "https://files.pythonhosted.org/packages/e9/dc/859d889b4d9356a1a2cdbc1e4a0dda94052bc5b5300098647e51a58c430b/ruff-0.8.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:729850feed82ef2440aa27946ab39c18cb4a8889c1128a6d589ffa028ddcfc22", size = 11220750 }, + { url = "https://files.pythonhosted.org/packages/0b/08/e8f519f61f1d624264bfd6b8829e4c5f31c3c61193bc3cff1f19dbe7626a/ruff-0.8.2-py3-none-win32.whl", hash = "sha256:ac42caaa0411d6a7d9594363294416e0e48fc1279e1b0e948391695db2b3d5b1", size = 8729396 }, + { url = "https://files.pythonhosted.org/packages/f8/d4/ba1c7ab72aba37a2b71fe48ab95b80546dbad7a7f35ea28cf66fc5cea5f6/ruff-0.8.2-py3-none-win_amd64.whl", hash = "sha256:2aae99ec70abf43372612a838d97bfe77d45146254568d94926e8ed5bbb409ea", size = 9594729 }, + { url = "https://files.pythonhosted.org/packages/23/34/db20e12d3db11b8a2a8874258f0f6d96a9a4d631659d54575840557164c8/ruff-0.8.2-py3-none-win_arm64.whl", hash = "sha256:fb88e2a506b70cfbc2de6fae6681c4f944f7dd5f2fe87233a7233d888bad73e8", size = 9035131 }, ] [[package]] @@ -2388,6 +2802,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl", hash = "sha256:a4468e1b37bb318f8a86514f65814e1afc977cf29b3992a4500d9dd305dcceb0", size = 14154 }, ] +[[package]] +name = "texttable" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/dc/0aff23d6036a4d3bf4f1d8c8204c5c79c4437e25e0ae94ffe4bbb55ee3c2/texttable-1.7.0.tar.gz", hash = "sha256:2d2068fb55115807d3ac77a4ca68fa48803e84ebb0ee2340f858107a36522638", size = 12831 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl", hash = "sha256:72227d592c82b3d7f672731ae73e4d1f88cd8e2ef5b075a7a7f01a23a3743917", size = 10768 }, +] + [[package]] name = "tinycss2" version = "1.3.0" @@ -2611,7 +3034,7 @@ dev = [ { name = "pytest", specifier = ">=6.2.4" }, { name = "pyyaml", specifier = ">=6.0.1" }, { name = "requests-mock", specifier = ">=1.9.3" }, - { name = "ruff", specifier = "==0.1.6" }, + { name = "ruff", specifier = "==0.8.2" }, { name = "tqdm", specifier = ">=4.62.3" }, { name = "types-pyyaml", specifier = ">=6.0.5" }, { name = "types-requests", specifier = ">=2.25.2" }, diff --git a/pyproject.toml b/pyproject.toml index 1cb685a3373..3e524a1b5ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,8 +32,7 @@ dependencies = [ "ruamel.yaml>=0.17.21", "gitpython>=3.1.30", "rapidfuzz>=2.13.7", - # pinned because of frictionless, remove this pin when frictionless is updated - "fsspec==2022.11.0", + "fsspec>=2022.11.0", "openai>=1.3.6", "pdfplumber>=0.9.0", "pyhumps>=3.8.0", @@ -58,14 +57,16 @@ dependencies = [ "earthengine-api>=0.1.411", "python-docx>=1.1.2", "h5netcdf>=1.3.0", - # we're not yet ready for frictionless 5.x.x as it raises an error for data://open_numbers/open_numbers/latest/bp__energy - "frictionless[pandas]>=4.40.8, <5.0.0", + "frictionless[pandas]>=5.0.3", "owid-catalog", "owid-datautils", "owid-repack", "walden", "deprecated>=1.2.14", "scikit-learn>=1.5.2", + "geopy>=2.4.1", + "py7zr>=0.22.0", + "pyreadr>=0.5.2", ] [tool.uv.sources] @@ -106,7 +107,7 @@ dev-dependencies = [ # unpinning those would introduce tons of type errors "pyright==1.1.373", "pandas-stubs==1.2.0.62", - "ruff==0.1.6", + "ruff>=0.8.2", "ipdb>=0.13.13", ] @@ -140,21 +141,23 @@ wizard = [ etl = 'apps.cli:cli' etlwiz = 'apps.wizard.cli:cli' etlr = 'etl.command:main_cli' +etlp = 'apps.pr.cli:cli' etl-wizard = 'apps.wizard.cli:cli' compare = 'etl.compare:cli' backport = 'apps.backport.backport:backport_cli' [tool.ruff] -extend-select = [ +lint.extend-select = [ # isort "I" ] -ignore = ["E501"] +lint.ignore = ["E501"] line-length = 120 target-version = "py310" extend-exclude = [ ".ipynb_checkpoints", "*cookiecutter", + "*.ipynb", ] [build-system] @@ -164,8 +167,6 @@ build-backend = "hatchling.build" [tool.pyright] exclude = [ "lib/", - "etl/steps/archive", - "etl/snapshots/archive", "apps/wizard/etl_steps/cookiecutter/", "apps/wizard/etl_steps/cookiecutter/snapshot/**", "**/node_modules", diff --git a/schemas/definitions.json b/schemas/definitions.json index f1b0ed711c2..42a7b2810d3 100644 --- a/schemas/definitions.json +++ b/schemas/definitions.json @@ -663,7 +663,19 @@ "description": "We keep display for the time being as the 'less powerful sibling' of grapher config.", "properties": { "isProjection": { - "type": "boolean", + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "pattern": "<%" + }, + { + "type": "string", + "pattern": "^\\{.*\\}$" + } + ], "default": false, "description": "Indicates if this time series is a forward projection (if so then this is rendered differently in e.g. line charts)." }, diff --git a/snapshots/antibiotics/2024-10-18/who_glass_by_antibiotic.py b/snapshots/antibiotics/2024-10-18/who_glass_by_antibiotic.py index cc02bd701bd..a9a0d52d0f4 100644 --- a/snapshots/antibiotics/2024-10-18/who_glass_by_antibiotic.py +++ b/snapshots/antibiotics/2024-10-18/who_glass_by_antibiotic.py @@ -13,7 +13,6 @@ """ - from pathlib import Path import click diff --git a/snapshots/antibiotics/2024-12-02/microbe_amr.csv.dvc b/snapshots/antibiotics/2024-12-02/microbe_amr.csv.dvc new file mode 100644 index 00000000000..3e75a57c69e --- /dev/null +++ b/snapshots/antibiotics/2024-12-02/microbe_amr.csv.dvc @@ -0,0 +1,31 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Neonatal deaths from infections attributed to antimicrobial resistance by syndrome + description: |- + The MICROBE (Measuring Infectious Causes and Resistance Outcomes for Burden Estimation) tool visualizes the fatal and nonfatal health outcomes of infections, pathogens, and antimicrobial resistance across different countries and regions. The tool shows a novel estimation method, [published in The Lancet](https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(24)01867-1/fulltext), including the burden of infections and their underlying pathogens, as well as, the burden of pathogens that are both susceptible and resistant to antibiotics. This tool is useful for understanding the burden of these outcomes, as well as illustrating how they nest together. The tabs explore different health outcomes by geography, age and sex. All tabs include a bar visualization for comparison, as well as a map view for a global perspective. + date_published: "2024-09-28" + + # Citation + producer: Institute for Health Metrics and Evaluation (IHME); University of Oxford + citation_full: |- + Institute for Health Metrics and Evaluation (IHME), University of Oxford. MICROBE. Seattle, WA: IHME, University of Washington, 2024. Available from [https://vizhub.healthdata.org/microbe](https://vizhub.healthdata.org/microbe) + attribution_short: MICROBE + + # Files + url_main: https://vizhub.healthdata.org/microbe/ + date_accessed: 2024-12-02 + + # License + license: + name: IHME's Free-of-Charge Non-commercial User Agreement + url: https://www.healthdata.org/Data-tools-practices/data-practices/ihme-free-charge-non-commercial-user-agreement + + + is_public: false +outs: + - md5: 0236cab87d4198c1ddb914b828c36eb0 + size: 3594 + path: microbe_amr.csv diff --git a/snapshots/antibiotics/2024-12-02/microbe_amr.py b/snapshots/antibiotics/2024-12-02/microbe_amr.py new file mode 100644 index 00000000000..be5d1cd965e --- /dev/null +++ b/snapshots/antibiotics/2024-12-02/microbe_amr.py @@ -0,0 +1,44 @@ +"""Script to create a snapshot of dataset. + +To access the data for this snapshot, follow these steps: + +- Go to https://vizhub.healthdata.org/microbe/ +- Click on Antimicrobial Resistance in the top tab +- Select these options: + - Category: Syndromes + - Burden: By resistance + - Location: Global + - Age: Neonatal + - Counterfactual: Both (or Attributable) + - Year: 2021 (or whatever latest year is available) + - Measure: Deaths + - Metric: Number + +Download the file and upload it using the script below. + + +""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +@click.option("--path-to-file", "-f", prompt=True, type=str, help="Path to local data file.") +def main(path_to_file: str, upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"antibiotics/{SNAPSHOT_VERSION}/microbe_amr.csv") + + # Copy local data file to snapshots data folder, add file to DVC and upload to S3. + snap.create_snapshot(filename=path_to_file, upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/antibiotics/2024-12-02/microbe_neonatal_amr.csv.dvc b/snapshots/antibiotics/2024-12-02/microbe_neonatal_amr.csv.dvc new file mode 100644 index 00000000000..c1e29c03deb --- /dev/null +++ b/snapshots/antibiotics/2024-12-02/microbe_neonatal_amr.csv.dvc @@ -0,0 +1,30 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Neonatal deaths from bloodstream infections attributed to antimicrobial resistance by pathogen + description: |- + The MICROBE (Measuring Infectious Causes and Resistance Outcomes for Burden Estimation) tool visualizes the fatal and nonfatal health outcomes of infections, pathogens, and antimicrobial resistance across different countries and regions. The tool shows a novel estimation method, [published in The Lancet](https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(24)01867-1/fulltext), including the burden of infections and their underlying pathogens, as well as, the burden of pathogens that are both susceptible and resistant to antibiotics. This tool is useful for understanding the burden of these outcomes, as well as illustrating how they nest together. The tabs explore different health outcomes by geography, age and sex. All tabs include a bar visualization for comparison, as well as a map view for a global perspective. + + date_published: "2024-09-28" + + # Citation + producer: Institute for Health Metrics and Evaluation (IHME); University of Oxford + citation_full: |- + Institute for Health Metrics and Evaluation (IHME), University of Oxford. MICROBE. Seattle, WA: IHME, University of Washington, 2024. Available from [https://vizhub.healthdata.org/microbe](https://vizhub.healthdata.org/microbe) + attribution_short: MICROBE + # Files + url_main: https://vizhub.healthdata.org/microbe/ + date_accessed: 2024-12-02 + + # License + license: + name: IHME's Free-of-Charge Non-commercial User Agreement + url: https://www.healthdata.org/Data-tools-practices/data-practices/ihme-free-charge-non-commercial-user-agreement + + is_public: false +outs: + - md5: 1e718707c8e293ef851d7baf0fef36a1 + size: 3014 + path: microbe_neonatal_amr.csv diff --git a/snapshots/antibiotics/2024-12-02/microbe_neonatal_amr.py b/snapshots/antibiotics/2024-12-02/microbe_neonatal_amr.py new file mode 100644 index 00000000000..19af79c2455 --- /dev/null +++ b/snapshots/antibiotics/2024-12-02/microbe_neonatal_amr.py @@ -0,0 +1,45 @@ +"""Script to create a snapshot of dataset. + +To access the data for this snapshot, follow these steps: + +- Go to https://vizhub.healthdata.org/microbe/ +- Click on Antimicrobial Resistance in the top tab +- Select these options: + - Category: Pathogens + - Burden: By resistance + - Infectious syndrome: Bloodstream infections + - Location: Global + - Age: Neonatal + - Counterfactual: Both (or Attributable) + - Year: 2021 (or whatever latest year is available) + - Measure: Deaths + - Metric: Number + +Download the file and upload it using the script below. + + +""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +@click.option("--path-to-file", "-f", prompt=True, type=str, help="Path to local data file.") +def main(path_to_file: str, upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"antibiotics/{SNAPSHOT_VERSION}/microbe_neonatal_amr.csv") + + # Copy local data file to snapshots data folder, add file to DVC and upload to S3. + snap.create_snapshot(filename=path_to_file, upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/antibiotics/2024-11-20/bloodstream_amr.csv.dvc b/snapshots/antibiotics/2024-12-02/total_pathogen_bloodstream.csv.dvc similarity index 89% rename from snapshots/antibiotics/2024-11-20/bloodstream_amr.csv.dvc rename to snapshots/antibiotics/2024-12-02/total_pathogen_bloodstream.csv.dvc index d697903762e..26192a4daf4 100644 --- a/snapshots/antibiotics/2024-11-20/bloodstream_amr.csv.dvc +++ b/snapshots/antibiotics/2024-12-02/total_pathogen_bloodstream.csv.dvc @@ -3,22 +3,20 @@ meta: origin: # Data product / Snapshot - title: Neonatal deaths from bloodstream infections by pathogen and resistance type + title: Total deaths from bloodstream infections by pathogen description: |- The MICROBE (Measuring Infectious Causes and Resistance Outcomes for Burden Estimation) tool visualizes the fatal and nonfatal health outcomes of infections, pathogens, and antimicrobial resistance across different countries and regions. The tool shows a novel estimation method, [published in The Lancet](https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(24)01867-1/fulltext), including the burden of infections and their underlying pathogens, as well as, the burden of pathogens that are both susceptible and resistant to antibiotics. This tool is useful for understanding the burden of these outcomes, as well as illustrating how they nest together. The tabs explore different health outcomes by geography, age and sex. All tabs include a bar visualization for comparison, as well as a map view for a global perspective. - date_published: "2024-09-28" # Citation producer: Institute for Health Metrics and Evaluation (IHME); University of Oxford citation_full: |- Institute for Health Metrics and Evaluation (IHME), University of Oxford. MICROBE. Seattle, WA: IHME, University of Washington, 2024. Available from [https://vizhub.healthdata.org/microbe](https://vizhub.healthdata.org/microbe) - attribution_short: MICROBE # Files url_main: https://vizhub.healthdata.org/microbe/ - date_accessed: 2024-11-20 + date_accessed: 2024-12-02 # License license: @@ -28,6 +26,6 @@ meta: is_public: false outs: - - md5: 56bf846bbb0e76403635ba94b784eedb - size: 9491 - path: bloodstream_amr.csv + - md5: 04ae50c86998acf6143a1685f04068ae + size: 6452 + path: total_pathogen_bloodstream.csv diff --git a/snapshots/antibiotics/2024-12-02/total_pathogen_bloodstream.py b/snapshots/antibiotics/2024-12-02/total_pathogen_bloodstream.py new file mode 100644 index 00000000000..564c409dd7c --- /dev/null +++ b/snapshots/antibiotics/2024-12-02/total_pathogen_bloodstream.py @@ -0,0 +1,43 @@ +"""Script to create a snapshot of dataset. + +To access the data for this snapshot, follow these steps: + +- Go to https://vizhub.healthdata.org/microbe/ +- Click on Pathogens in the top tab +- Select these options: + - Infectious syndrome: Bloodsteam infections + - Location: Global + - Age: All ages + - Sex: Both + - Year: 2021 (or whatever latest year is available) + - Measure: Deaths + - Metric: Number + +Download the file and upload it using the script below. + + +""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +@click.option("--path-to-file", "-f", prompt=True, type=str, help="Path to local data file.") +def main(path_to_file: str, upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"antibiotics/{SNAPSHOT_VERSION}/total_pathogen_bloodstream.csv") + + # Copy local data file to snapshots data folder, add file to DVC and upload to S3. + snap.create_snapshot(filename=path_to_file, upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.csv.dvc b/snapshots/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.csv.dvc new file mode 100644 index 00000000000..8225476e843 --- /dev/null +++ b/snapshots/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.csv.dvc @@ -0,0 +1,31 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Total deaths from bloodstream infections by pathogen and resistance + description: |- + The MICROBE (Measuring Infectious Causes and Resistance Outcomes for Burden Estimation) tool visualizes the fatal and nonfatal health outcomes of infections, pathogens, and antimicrobial resistance across different countries and regions. The tool shows a novel estimation method, [published in The Lancet](https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(24)01867-1/fulltext), including the burden of infections and their underlying pathogens, as well as, the burden of pathogens that are both susceptible and resistant to antibiotics. This tool is useful for understanding the burden of these outcomes, as well as illustrating how they nest together. The tabs explore different health outcomes by geography, age and sex. All tabs include a bar visualization for comparison, as well as a map view for a global perspective. + date_published: "2024-09-28" + + # Citation + producer: Institute for Health Metrics and Evaluation (IHME); University of Oxford + citation_full: |- + Institute for Health Metrics and Evaluation (IHME), University of Oxford. MICROBE. Seattle, WA: IHME, University of Washington, 2024. Available from [https://vizhub.healthdata.org/microbe](https://vizhub.healthdata.org/microbe) + attribution_short: MICROBE + + # Files + url_main: https://vizhub.healthdata.org/microbe/ + date_accessed: 2024-12-02 + + # License + license: + name: IHME's Free-of-Charge Non-commercial User Agreement + url: https://www.healthdata.org/Data-tools-practices/data-practices/ihme-free-charge-non-commercial-user-agreement + + + is_public: false +outs: + - md5: d76d22fd3288eccd20d838712105d773 + size: 3006 + path: total_pathogen_bloodstream_amr.csv diff --git a/snapshots/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.py b/snapshots/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.py new file mode 100644 index 00000000000..c021512e369 --- /dev/null +++ b/snapshots/antibiotics/2024-12-02/total_pathogen_bloodstream_amr.py @@ -0,0 +1,25 @@ +"""Script to create a snapshot of dataset.""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +@click.option("--path-to-file", "-f", prompt=True, type=str, help="Path to local data file.") +def main(path_to_file: str, upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"antibiotics/{SNAPSHOT_VERSION}/total_pathogen_bloodstream_amr.csv") + + # Copy local data file to snapshots data folder, add file to DVC and upload to S3. + snap.create_snapshot(filename=path_to_file, upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/antibiotics/2024-12-03/glass_enrolment.py b/snapshots/antibiotics/2024-12-03/glass_enrolment.py new file mode 100644 index 00000000000..9c31f08a086 --- /dev/null +++ b/snapshots/antibiotics/2024-12-03/glass_enrolment.py @@ -0,0 +1,30 @@ +"""Script to create a snapshot of dataset. + +The data should be available here: https://www.who.int/initiatives/glass/country-participation + +But if it is out of date (e.g not in sync with the image on the page above), then contact glass@who.int to access the latest data. +""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +@click.option("--path-to-file", "-f", prompt=True, type=str, help="Path to local data file.") +def main(path_to_file: str, upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"antibiotics/{SNAPSHOT_VERSION}/glass_enrolment.xlsx") + + # Copy local data file to snapshots data folder, add file to DVC and upload to S3. + snap.create_snapshot(filename=path_to_file, upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/antibiotics/2024-12-03/glass_enrolment.xlsx.dvc b/snapshots/antibiotics/2024-12-03/glass_enrolment.xlsx.dvc new file mode 100644 index 00000000000..a124b134985 --- /dev/null +++ b/snapshots/antibiotics/2024-12-03/glass_enrolment.xlsx.dvc @@ -0,0 +1,29 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: GLASS Country Participation + description: |- + The list of countries that are enrolled in the WHO's Global Antimicrobial Resistance and Use Surveillance System (GLASS). Countries can be enrolled in GLASS to collect and share data on antimicrobial consumption (AMC), and, or antimicrobial resistance (AMR) in line with the GLASS-AMR and GLASS-AMC methodologies, respectively. + date_published: "2024-12-03" + + # Citation + producer: World Health Organization + citation_full: |- + GLASS Country Participation (2024). Global Antimicrobial Resistance and Use Surveillance System (GLASS), World Health Organization. + attribution_short: WHO + + # Files + url_main: https://www.who.int/initiatives/glass/country-participation + date_accessed: 2024-12-03 + + # License + license: + name: © 2024 WHO + url: https://www.who.int/about/policies/terms-of-use + +outs: + - md5: d10cca0830352c8145c9feb4958b4120 + size: 88534 + path: glass_enrolment.xlsx diff --git a/snapshots/antibiotics/2024-12-04/microbe_total_pathogens.csv.dvc b/snapshots/antibiotics/2024-12-04/microbe_total_pathogens.csv.dvc new file mode 100644 index 00000000000..e904e5d6873 --- /dev/null +++ b/snapshots/antibiotics/2024-12-04/microbe_total_pathogens.csv.dvc @@ -0,0 +1,27 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Total deaths by pathogen + description: |- + The MICROBE (Measuring Infectious Causes and Resistance Outcomes for Burden Estimation) tool visualizes the fatal and nonfatal health outcomes of infections, pathogens, and antimicrobial resistance across different countries and regions. The tool shows a novel estimation method, [published in The Lancet](https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(24)01867-1/fulltext), including the burden of infections and their underlying pathogens, as well as, the burden of pathogens that are both susceptible and resistant to antibiotics. This tool is useful for understanding the burden of these outcomes, as well as illustrating how they nest together. The tabs explore different health outcomes by geography, age and sex. All tabs include a bar visualization for comparison, as well as a map view for a global perspective. + date_published: "2024-09-28" + # Citation + producer: Institute for Health Metrics and Evaluation (IHME); University of Oxford + citation_full: |- + Institute for Health Metrics and Evaluation (IHME), University of Oxford. MICROBE. Seattle, WA: IHME, University of Washington, 2024. Available from [https://vizhub.healthdata.org/microbe](https://vizhub.healthdata.org/microbe) + attribution_short: MICROBE + # Files + url_main: https://vizhub.healthdata.org/microbe/ + date_accessed: 2024-12-04 + + # License + license: + name: IHME's Free-of-Charge Non-commercial User Agreement + url: https://www.healthdata.org/Data-tools-practices/data-practices/ihme-free-charge-non-commercial-user-agreement + is_public: false +outs: + - md5: bff21259d44b1ab5a61ecf675285d13b + size: 10960 + path: microbe_total_pathogens.csv diff --git a/snapshots/antibiotics/2024-12-04/microbe_total_pathogens.py b/snapshots/antibiotics/2024-12-04/microbe_total_pathogens.py new file mode 100644 index 00000000000..754aaafe897 --- /dev/null +++ b/snapshots/antibiotics/2024-12-04/microbe_total_pathogens.py @@ -0,0 +1,37 @@ +"""Script to create a snapshot of dataset. + +To download the data visit: https://vizhub.healthdata.org/microbe/ + +- Select the 'Pathogens' tab. +- Infectious syndrome: 'All infectious syndromes' +- Location: 'Global' +- Age: 'All ages' +- Sex: 'Both' +- Measure: 'Deaths' +- Metric: 'Number' + +""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +@click.option("--path-to-file", "-f", prompt=True, type=str, help="Path to local data file.") +def main(path_to_file: str, upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"antibiotics/{SNAPSHOT_VERSION}/microbe_total_pathogens.csv") + + # Download data from source, add file to DVC and upload to S3. + snap.create_snapshot(filename=path_to_file, upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/antibiotics/2024-12-04/microbe_total_pathogens_amr.csv.dvc b/snapshots/antibiotics/2024-12-04/microbe_total_pathogens_amr.csv.dvc new file mode 100644 index 00000000000..77cd02a9cf8 --- /dev/null +++ b/snapshots/antibiotics/2024-12-04/microbe_total_pathogens_amr.csv.dvc @@ -0,0 +1,31 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Total deaths by pathogen attributable to antimicrobial resistance + description: |- + The MICROBE (Measuring Infectious Causes and Resistance Outcomes for Burden Estimation) tool visualizes the fatal and nonfatal health outcomes of infections, pathogens, and antimicrobial resistance across different countries and regions. The tool shows a novel estimation method, [published in The Lancet](https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(24)01867-1/fulltext), including the burden of infections and their underlying pathogens, as well as, the burden of pathogens that are both susceptible and resistant to antibiotics. This tool is useful for understanding the burden of these outcomes, as well as illustrating how they nest together. The tabs explore different health outcomes by geography, age and sex. All tabs include a bar visualization for comparison, as well as a map view for a global perspective. + date_published: "2024-09-28" + + # Citation + producer: Institute for Health Metrics and Evaluation (IHME); University of Oxford + citation_full: |- + Institute for Health Metrics and Evaluation (IHME), University of Oxford. MICROBE. Seattle, WA: IHME, University of Washington, 2024. Available from [https://vizhub.healthdata.org/microbe](https://vizhub.healthdata.org/microbe) + attribution_short: MICROBE + + # Files + url_main: https://vizhub.healthdata.org/microbe/ + date_accessed: 2024-12-04 + + # License + license: + name: IHME's Free-of-Charge Non-commercial User Agreement + url: https://www.healthdata.org/Data-tools-practices/data-practices/ihme-free-charge-non-commercial-user-agreement + + + is_public: false +outs: + - md5: 95bd7ca4c721a4e5113fd54ee598dad3 + size: 3989 + path: microbe_total_pathogens_amr.csv diff --git a/snapshots/antibiotics/2024-12-04/microbe_total_pathogens_amr.py b/snapshots/antibiotics/2024-12-04/microbe_total_pathogens_amr.py new file mode 100644 index 00000000000..7b387e0ef02 --- /dev/null +++ b/snapshots/antibiotics/2024-12-04/microbe_total_pathogens_amr.py @@ -0,0 +1,25 @@ +"""Script to create a snapshot of dataset.""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +@click.option("--path-to-file", "-f", prompt=True, type=str, help="Path to local data file.") +def main(path_to_file: str, upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"antibiotics/{SNAPSHOT_VERSION}/microbe_total_pathogens_amr.csv") + + # Copy local data file to snapshots data folder, add file to DVC and upload to S3. + snap.create_snapshot(filename=path_to_file, upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/antibiotics/2024-12-05/microbe_neonatal_total_amr.csv.dvc b/snapshots/antibiotics/2024-12-05/microbe_neonatal_total_amr.csv.dvc new file mode 100644 index 00000000000..357627600e7 --- /dev/null +++ b/snapshots/antibiotics/2024-12-05/microbe_neonatal_total_amr.csv.dvc @@ -0,0 +1,31 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Neonatal deaths by pathogen attributable to antimicrobial resistance + description: |- + The MICROBE (Measuring Infectious Causes and Resistance Outcomes for Burden Estimation) tool visualizes the fatal and nonfatal health outcomes of infections, pathogens, and antimicrobial resistance across different countries and regions. The tool shows a novel estimation method, [published in The Lancet](https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(24)01867-1/fulltext), including the burden of infections and their underlying pathogens, as well as, the burden of pathogens that are both susceptible and resistant to antibiotics. This tool is useful for understanding the burden of these outcomes, as well as illustrating how they nest together. The tabs explore different health outcomes by geography, age and sex. All tabs include a bar visualization for comparison, as well as a map view for a global perspective. + date_published: "2024-09-28" + + # Citation + producer: Institute for Health Metrics and Evaluation (IHME); University of Oxford + citation_full: |- + Institute for Health Metrics and Evaluation (IHME), University of Oxford. MICROBE. Seattle, WA: IHME, University of Washington, 2024. Available from [https://vizhub.healthdata.org/microbe](https://vizhub.healthdata.org/microbe) + attribution_short: MICROBE + + # Files + url_main: https://vizhub.healthdata.org/microbe/ + date_accessed: 2024-12-05 + + # License + license: + name: IHME's Free-of-Charge Non-commercial User Agreement + url: https://www.healthdata.org/Data-tools-practices/data-practices/ihme-free-charge-non-commercial-user-agreement + + + is_public: false +outs: + - md5: 992049355ab469d34378e2ae38ff2ef5 + size: 3919 + path: microbe_neonatal_total_amr.csv diff --git a/snapshots/antibiotics/2024-12-05/microbe_neonatal_total_amr.py b/snapshots/antibiotics/2024-12-05/microbe_neonatal_total_amr.py new file mode 100644 index 00000000000..530324b4376 --- /dev/null +++ b/snapshots/antibiotics/2024-12-05/microbe_neonatal_total_amr.py @@ -0,0 +1,37 @@ +"""Script to create a snapshot of dataset. + +To download the data visit: https://vizhub.healthdata.org/microbe/ + +- Select the 'Antimicrobial resistance' tab. +- Cateogory: 'Pathogens' +- Location: 'Global' +- Age: 'Neonatal' +- Counterfactual: 'Attributable' +- Measure: 'Deaths' +- Metric: 'Number' + +""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +@click.option("--path-to-file", "-f", prompt=True, type=str, help="Path to local data file.") +def main(path_to_file: str, upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"antibiotics/{SNAPSHOT_VERSION}/microbe_neonatal_total_amr.csv") + + # Copy local data file to snapshots data folder, add file to DVC and upload to S3. + snap.create_snapshot(filename=path_to_file, upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.csv.dvc b/snapshots/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.csv.dvc new file mode 100644 index 00000000000..bb61d4c9241 --- /dev/null +++ b/snapshots/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.csv.dvc @@ -0,0 +1,28 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Total deaths by syndrome + description: |- + The MICROBE (Measuring Infectious Causes and Resistance Outcomes for Burden Estimation) tool visualizes the fatal and nonfatal health outcomes of infections, pathogens, and antimicrobial resistance across different countries and regions. The tool shows a novel estimation method, [published in The Lancet](https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(24)01867-1/fulltext), including the burden of infections and their underlying pathogens, as well as, the burden of pathogens that are both susceptible and resistant to antibiotics. This tool is useful for understanding the burden of these outcomes, as well as illustrating how they nest together. The tabs explore different health outcomes by geography, age and sex. All tabs include a bar visualization for comparison, as well as a map view for a global perspective. + date_published: "2024-09-28" + # Citation + producer: Institute for Health Metrics and Evaluation (IHME); University of Oxford + citation_full: |- + Institute for Health Metrics and Evaluation (IHME), University of Oxford. MICROBE. Seattle, WA: IHME, University of Washington, 2024. Available from [https://vizhub.healthdata.org/microbe](https://vizhub.healthdata.org/microbe) + attribution_short: MICROBE + # Files + url_main: https://vizhub.healthdata.org/microbe/ + date_accessed: 2024-12-05 + + # License + license: + name: IHME's Free-of-Charge Non-commercial User Agreement + url: https://www.healthdata.org/Data-tools-practices/data-practices/ihme-free-charge-non-commercial-user-agreement + + is_public: false +outs: + - md5: d494bc6e6c0a61cf05d3d9fb03a2f0b5 + size: 3454 + path: microbe_total_deaths_by_syndrome.csv diff --git a/snapshots/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.py b/snapshots/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.py new file mode 100644 index 00000000000..2006c8a692f --- /dev/null +++ b/snapshots/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.py @@ -0,0 +1,37 @@ +"""Script to create a snapshot of dataset. + +To download the data visit: https://vizhub.healthdata.org/microbe/ + +- Select the 'Antimicrobial resistance' tab. +- Cateogory: 'Pathogens' +- Location: 'Global' +- Age: 'Neonatal' +- Counterfactual: 'Attributable' +- Measure: 'Deaths' +- Metric: 'Number' + +""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +@click.option("--path-to-file", "-f", prompt=True, type=str, help="Path to local data file.") +def main(path_to_file: str, upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"antibiotics/{SNAPSHOT_VERSION}/microbe_total_deaths_by_syndrome.csv") + + # Copy local data file to snapshots data folder, add file to DVC and upload to S3. + snap.create_snapshot(filename=path_to_file, upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.csv.dvc b/snapshots/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.csv.dvc new file mode 100644 index 00000000000..f845b6df7bd --- /dev/null +++ b/snapshots/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.csv.dvc @@ -0,0 +1,28 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Total deaths by syndrome and antimicrobial resistance + description: |- + The MICROBE (Measuring Infectious Causes and Resistance Outcomes for Burden Estimation) tool visualizes the fatal and nonfatal health outcomes of infections, pathogens, and antimicrobial resistance across different countries and regions. The tool shows a novel estimation method, [published in The Lancet](https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(24)01867-1/fulltext), including the burden of infections and their underlying pathogens, as well as, the burden of pathogens that are both susceptible and resistant to antibiotics. This tool is useful for understanding the burden of these outcomes, as well as illustrating how they nest together. The tabs explore different health outcomes by geography, age and sex. All tabs include a bar visualization for comparison, as well as a map view for a global perspective. + date_published: "2024-09-28" + # Citation + producer: Institute for Health Metrics and Evaluation (IHME); University of Oxford + citation_full: |- + Institute for Health Metrics and Evaluation (IHME), University of Oxford. MICROBE. Seattle, WA: IHME, University of Washington, 2024. Available from [https://vizhub.healthdata.org/microbe](https://vizhub.healthdata.org/microbe) + attribution_short: MICROBE + # Files + url_main: https://vizhub.healthdata.org/microbe/ + date_accessed: 2024-12-05 + + # License + license: + name: IHME's Free-of-Charge Non-commercial User Agreement + url: https://www.healthdata.org/Data-tools-practices/data-practices/ihme-free-charge-non-commercial-user-agreement + + is_public: false +outs: + - md5: fe3b1da5275f0a31e882fab841b32ee4 + size: 1860 + path: microbe_total_deaths_by_syndrome_amr.csv diff --git a/snapshots/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.py b/snapshots/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.py new file mode 100644 index 00000000000..fbb2dc82667 --- /dev/null +++ b/snapshots/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.py @@ -0,0 +1,37 @@ +"""Script to create a snapshot of dataset. + +To download the data visit: https://vizhub.healthdata.org/microbe/ + +- Select the 'Antimicrobial resistance' tab. +- Cateogory: 'Pathogens' +- Location: 'Global' +- Age: 'Neonatal' +- Counterfactual: 'Attributable' +- Measure: 'Deaths' +- Metric: 'Number' + +""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +@click.option("--path-to-file", "-f", prompt=True, type=str, help="Path to local data file.") +def main(path_to_file: str, upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"antibiotics/{SNAPSHOT_VERSION}/microbe_total_deaths_by_syndrome_amr.csv") + + # Copy local data file to snapshots data folder, add file to DVC and upload to S3. + snap.create_snapshot(filename=path_to_file, upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/artificial_intelligence/2023-06-21/epoch.py b/snapshots/artificial_intelligence/2023-06-21/epoch.py index daa355e267f..25991e18d5d 100644 --- a/snapshots/artificial_intelligence/2023-06-21/epoch.py +++ b/snapshots/artificial_intelligence/2023-06-21/epoch.py @@ -1,6 +1,5 @@ """Script to create a snapshot of dataset 'Parameter, Compute and Data Trends in Machine Learning (Epoch, 2023)'.""" - from pathlib import Path import click diff --git a/snapshots/artificial_intelligence/2024-06-03/epoch.py b/snapshots/artificial_intelligence/2024-06-03/epoch.py index daa355e267f..25991e18d5d 100644 --- a/snapshots/artificial_intelligence/2024-06-03/epoch.py +++ b/snapshots/artificial_intelligence/2024-06-03/epoch.py @@ -1,6 +1,5 @@ """Script to create a snapshot of dataset 'Parameter, Compute and Data Trends in Machine Learning (Epoch, 2023)'.""" - from pathlib import Path import click diff --git a/snapshots/artificial_intelligence/2024-07-10/epoch.py b/snapshots/artificial_intelligence/2024-07-10/epoch.py index daa355e267f..25991e18d5d 100644 --- a/snapshots/artificial_intelligence/2024-07-10/epoch.py +++ b/snapshots/artificial_intelligence/2024-07-10/epoch.py @@ -1,6 +1,5 @@ """Script to create a snapshot of dataset 'Parameter, Compute and Data Trends in Machine Learning (Epoch, 2023)'.""" - from pathlib import Path import click diff --git a/snapshots/artificial_intelligence/2024-08-05/epoch.py b/snapshots/artificial_intelligence/2024-08-05/epoch.py index daa355e267f..25991e18d5d 100644 --- a/snapshots/artificial_intelligence/2024-08-05/epoch.py +++ b/snapshots/artificial_intelligence/2024-08-05/epoch.py @@ -1,6 +1,5 @@ """Script to create a snapshot of dataset 'Parameter, Compute and Data Trends in Machine Learning (Epoch, 2023)'.""" - from pathlib import Path import click diff --git a/snapshots/artificial_intelligence/2024-09-09/epoch.py b/snapshots/artificial_intelligence/2024-09-09/epoch.py index daa355e267f..25991e18d5d 100644 --- a/snapshots/artificial_intelligence/2024-09-09/epoch.py +++ b/snapshots/artificial_intelligence/2024-09-09/epoch.py @@ -1,6 +1,5 @@ """Script to create a snapshot of dataset 'Parameter, Compute and Data Trends in Machine Learning (Epoch, 2023)'.""" - from pathlib import Path import click diff --git a/snapshots/artificial_intelligence/2024-10-01/epoch.py b/snapshots/artificial_intelligence/2024-10-01/epoch.py index daa355e267f..25991e18d5d 100644 --- a/snapshots/artificial_intelligence/2024-10-01/epoch.py +++ b/snapshots/artificial_intelligence/2024-10-01/epoch.py @@ -1,6 +1,5 @@ """Script to create a snapshot of dataset 'Parameter, Compute and Data Trends in Machine Learning (Epoch, 2023)'.""" - from pathlib import Path import click diff --git a/snapshots/artificial_intelligence/2024-11-03/epoch.py b/snapshots/artificial_intelligence/2024-11-03/epoch.py index daa355e267f..25991e18d5d 100644 --- a/snapshots/artificial_intelligence/2024-11-03/epoch.py +++ b/snapshots/artificial_intelligence/2024-11-03/epoch.py @@ -1,6 +1,5 @@ """Script to create a snapshot of dataset 'Parameter, Compute and Data Trends in Machine Learning (Epoch, 2023)'.""" - from pathlib import Path import click diff --git a/snapshots/artificial_intelligence/2024-12-05/epoch.csv.dvc b/snapshots/artificial_intelligence/2024-12-05/epoch.csv.dvc new file mode 100644 index 00000000000..2ba13908db0 --- /dev/null +++ b/snapshots/artificial_intelligence/2024-12-05/epoch.csv.dvc @@ -0,0 +1,38 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/origin/ +meta: + origin: + # Data product / Snapshot + title: Parameter, Compute and Data Trends in Machine Learning + date_published: 2024-07-19 + description_snapshot: | + We update this chart with the latest available data from our source every month. + + The authors selected the AI systems for inclusion based on the following necessary criteria: + — Have an explicit learning component + — Showcase experimental results + — Advance the state of the art + + In addition, the systems had to meet at least one of the following notability criteria: + — Paper has more than 1000 citations + — Historical importance + — Important state-of-the-art advance + — Deployed in a notable context + + The authors note that: "For new models (from 2020 onward) it is harder to assess these criteria, so we fall back to a subjective selection. We refer to models meeting our selection criteria as 'milestone models." + # Citation + producer: Epoch + citation_full: "Epoch AI, ‘Parameter, Compute and Data Trends in Machine Learning’. Published online at epochai.org. Retrieved from: ‘https://epoch.ai/data/epochdb/visualization’ [online resource]" + # Files + url_main: https://epoch.ai/mlinputs/visualization + url_download: https://epoch.ai/data/epochdb/notable_ai_models.csv + date_accessed: 2024-12-05 + # License + license: + name: CC BY 4.0 + url: https://creativecommons.org/licenses/by/4.0/ + +outs: + - md5: 98750b0d23c2f5e11b766e0849432fb3 + size: 1600590 + path: epoch.csv diff --git a/snapshots/artificial_intelligence/2024-12-05/epoch.py b/snapshots/artificial_intelligence/2024-12-05/epoch.py new file mode 100644 index 00000000000..25991e18d5d --- /dev/null +++ b/snapshots/artificial_intelligence/2024-12-05/epoch.py @@ -0,0 +1,32 @@ +"""Script to create a snapshot of dataset 'Parameter, Compute and Data Trends in Machine Learning (Epoch, 2023)'.""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option( + "--upload/--skip-upload", + default=True, + type=bool, + help="Upload dataset to Snapshot", +) +def main(upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"artificial_intelligence/{SNAPSHOT_VERSION}/epoch.csv") + + # Download data from source. + snap.download_from_source() + + # Add file to DVC and upload to S3. + snap.dvc_add(upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/artificial_intelligence/2024-12-05/epoch_compute_intensive.csv.dvc b/snapshots/artificial_intelligence/2024-12-05/epoch_compute_intensive.csv.dvc new file mode 100644 index 00000000000..1850e7f75b3 --- /dev/null +++ b/snapshots/artificial_intelligence/2024-12-05/epoch_compute_intensive.csv.dvc @@ -0,0 +1,33 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Tracking Compute-Intensive AI Models + description: |- + A dataset that tracks compute-intensive AI models, with training compute over 10²³ floating point operations (FLOP). This corresponds to training costs of hundreds of thousands of dollars or more.  + + To identify compute-intensive AI models, the team at Epoch AI used various resources, estimating compute when not directly reported. They included benchmarks and repositories, such as Papers With Code and Hugging Face, to find models exceeding 10²³ FLOP. They also explored non-English media and specific leaderboards, particularly focusing on Chinese sources. + + Additionally, they examined blog posts, press releases from major labs, and scholarly literature to track new models. A separate table was created for models with unconfirmed but plausible compute levels. Despite thorough methods, proprietary and secretive models may have been missed. + date_published: "2024-06-19" + + # Citation + producer: Epoch + citation_full: |- + Robi Rahman, David Owen and Josh You (2024), "Tracking Compute-Intensive AI Models". Published online at epochai.org. Retrieved from: 'https://epoch.ai/blog/tracking-compute-intensive-ai-models' [online resource] + + # Files + url_main: https://epoch.ai/blog/tracking-compute-intensive-ai-models + url_download: https://epoch.ai/data/epochdb/large_scale_ai_models.csv + date_accessed: 2024-12-05 + + # License + license: + name: CC BY 4.0 + url: https://epoch.ai/blog/how-much-does-it-cost-to-train-frontier-ai-models + +outs: + - md5: c52df75e59048128dc8288a0467f3f4c + size: 484868 + path: epoch_compute_intensive.csv diff --git a/snapshots/artificial_intelligence/2024-12-05/epoch_compute_intensive.py b/snapshots/artificial_intelligence/2024-12-05/epoch_compute_intensive.py new file mode 100644 index 00000000000..fdbd7822e4a --- /dev/null +++ b/snapshots/artificial_intelligence/2024-12-05/epoch_compute_intensive.py @@ -0,0 +1,24 @@ +"""Script to create a snapshot of dataset.""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +def main(upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"artificial_intelligence/{SNAPSHOT_VERSION}/epoch_compute_intensive.csv") + + # Download data from source, add file to DVC and upload to S3. + snap.create_snapshot(upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/artificial_intelligence/latest/epoch.py b/snapshots/artificial_intelligence/latest/epoch.py index daa355e267f..25991e18d5d 100644 --- a/snapshots/artificial_intelligence/latest/epoch.py +++ b/snapshots/artificial_intelligence/latest/epoch.py @@ -1,6 +1,5 @@ """Script to create a snapshot of dataset 'Parameter, Compute and Data Trends in Machine Learning (Epoch, 2023)'.""" - from pathlib import Path import click diff --git a/snapshots/aviation_safety_network/2022-10-14/aviation_statistics.py b/snapshots/aviation_safety_network/2022-10-14/aviation_statistics.py index 3fdcba48fa7..ee574dc5b0e 100644 --- a/snapshots/aviation_safety_network/2022-10-14/aviation_statistics.py +++ b/snapshots/aviation_safety_network/2022-10-14/aviation_statistics.py @@ -1,6 +1,4 @@ -"""Get data from the Aviation Safety Network (extracted from a Google Sheet). - -""" +"""Get data from the Aviation Safety Network (extracted from a Google Sheet).""" import click import pandas as pd diff --git a/snapshots/cancer/2024-10-13/gco_cancer_over_time_cervical.py b/snapshots/cancer/2024-10-13/gco_cancer_over_time_cervical.py index 2c477a9ba5e..9187ec1c63f 100644 --- a/snapshots/cancer/2024-10-13/gco_cancer_over_time_cervical.py +++ b/snapshots/cancer/2024-10-13/gco_cancer_over_time_cervical.py @@ -2,6 +2,7 @@ To download the csv file go to https://gco.iarc.fr/overtime/en/dataviz/trends?populations=752_32_36_40_48_112_124_152_156_170_188_191_196_203_208_218_233_246_250_276_352_356_372_380_376_392_414_428_410_440_470_474_528_554_578_608_616_630_634_705_724_756_764_792_800_840_8260_8261_8262_8263_8401_8402&sexes=2&types=0&multiple_populations=1&group_populations=0&cancers=16 and click on the Downloads button --> csv. """ + from pathlib import Path import click diff --git a/snapshots/cancer/2024-10-13/gco_cancer_today_cervical.py b/snapshots/cancer/2024-10-13/gco_cancer_today_cervical.py index 8f8e4b14ad4..05613eac98a 100644 --- a/snapshots/cancer/2024-10-13/gco_cancer_today_cervical.py +++ b/snapshots/cancer/2024-10-13/gco_cancer_today_cervical.py @@ -1,5 +1,6 @@ """Script to create a snapshot of dataset. To download the csv file go to https://gco.iarc.fr/today/en/dataviz/maps-heatmap?mode=population&cancers=23&sexes=2 and click Downloads --> csv""" + from pathlib import Path import click diff --git a/snapshots/climate/2023-12-20/surface_temperature.py b/snapshots/climate/2023-12-20/surface_temperature.py index 68c70d98187..0ae58ec999a 100644 --- a/snapshots/climate/2023-12-20/surface_temperature.py +++ b/snapshots/climate/2023-12-20/surface_temperature.py @@ -1,12 +1,12 @@ """Script to create a snapshot of the monthly averaged surface temperature data from 1950 to present from the Copernicus Climate Change Service. - The script assumes that the data is available on the CDS API. - Instructions on how to access the API on a Mac are here: https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+macOS +The script assumes that the data is available on the CDS API. +Instructions on how to access the API on a Mac are here: https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+macOS - More information on how to access the data is here: https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview +More information on how to access the data is here: https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview - The data is downloaded as a NetCDF file. Tutorials for using the Copernicus API are here and work with the NETCDF format are here: https://ecmwf-projects.github.io/copernicus-training-c3s/cds-tutorial.html - """ +The data is downloaded as a NetCDF file. Tutorials for using the Copernicus API are here and work with the NETCDF format are here: https://ecmwf-projects.github.io/copernicus-training-c3s/cds-tutorial.html +""" import gzip import shutil diff --git a/snapshots/climate/2024-02-14/surface_temperature.py b/snapshots/climate/2024-02-14/surface_temperature.py index 3cc66fcba72..d30b45a7c24 100644 --- a/snapshots/climate/2024-02-14/surface_temperature.py +++ b/snapshots/climate/2024-02-14/surface_temperature.py @@ -1,12 +1,12 @@ """Script to create a snapshot of the monthly averaged surface temperature data from 1950 to present from the Copernicus Climate Change Service. - The script assumes that the data is available on the CDS API. - Instructions on how to access the API on a Mac are here: https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+macOS +The script assumes that the data is available on the CDS API. +Instructions on how to access the API on a Mac are here: https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+macOS - More information on how to access the data is here: https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview +More information on how to access the data is here: https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview - The data is downloaded as a NetCDF file. Tutorials for using the Copernicus API are here and work with the NETCDF format are here: https://ecmwf-projects.github.io/copernicus-training-c3s/cds-tutorial.html - """ +The data is downloaded as a NetCDF file. Tutorials for using the Copernicus API are here and work with the NETCDF format are here: https://ecmwf-projects.github.io/copernicus-training-c3s/cds-tutorial.html +""" import gzip import shutil diff --git a/snapshots/climate/2024-03-12/surface_temperature.py b/snapshots/climate/2024-03-12/surface_temperature.py index 3cc66fcba72..d30b45a7c24 100644 --- a/snapshots/climate/2024-03-12/surface_temperature.py +++ b/snapshots/climate/2024-03-12/surface_temperature.py @@ -1,12 +1,12 @@ """Script to create a snapshot of the monthly averaged surface temperature data from 1950 to present from the Copernicus Climate Change Service. - The script assumes that the data is available on the CDS API. - Instructions on how to access the API on a Mac are here: https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+macOS +The script assumes that the data is available on the CDS API. +Instructions on how to access the API on a Mac are here: https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+macOS - More information on how to access the data is here: https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview +More information on how to access the data is here: https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview - The data is downloaded as a NetCDF file. Tutorials for using the Copernicus API are here and work with the NETCDF format are here: https://ecmwf-projects.github.io/copernicus-training-c3s/cds-tutorial.html - """ +The data is downloaded as a NetCDF file. Tutorials for using the Copernicus API are here and work with the NETCDF format are here: https://ecmwf-projects.github.io/copernicus-training-c3s/cds-tutorial.html +""" import gzip import shutil diff --git a/snapshots/climate/2024-04-12/surface_temperature.py b/snapshots/climate/2024-04-12/surface_temperature.py index 3cc66fcba72..d30b45a7c24 100644 --- a/snapshots/climate/2024-04-12/surface_temperature.py +++ b/snapshots/climate/2024-04-12/surface_temperature.py @@ -1,12 +1,12 @@ """Script to create a snapshot of the monthly averaged surface temperature data from 1950 to present from the Copernicus Climate Change Service. - The script assumes that the data is available on the CDS API. - Instructions on how to access the API on a Mac are here: https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+macOS +The script assumes that the data is available on the CDS API. +Instructions on how to access the API on a Mac are here: https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+macOS - More information on how to access the data is here: https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview +More information on how to access the data is here: https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview - The data is downloaded as a NetCDF file. Tutorials for using the Copernicus API are here and work with the NETCDF format are here: https://ecmwf-projects.github.io/copernicus-training-c3s/cds-tutorial.html - """ +The data is downloaded as a NetCDF file. Tutorials for using the Copernicus API are here and work with the NETCDF format are here: https://ecmwf-projects.github.io/copernicus-training-c3s/cds-tutorial.html +""" import gzip import shutil diff --git a/snapshots/climate/2024-05-07/surface_temperature.py b/snapshots/climate/2024-05-07/surface_temperature.py index 3cc66fcba72..d30b45a7c24 100644 --- a/snapshots/climate/2024-05-07/surface_temperature.py +++ b/snapshots/climate/2024-05-07/surface_temperature.py @@ -1,12 +1,12 @@ """Script to create a snapshot of the monthly averaged surface temperature data from 1950 to present from the Copernicus Climate Change Service. - The script assumes that the data is available on the CDS API. - Instructions on how to access the API on a Mac are here: https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+macOS +The script assumes that the data is available on the CDS API. +Instructions on how to access the API on a Mac are here: https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+macOS - More information on how to access the data is here: https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview +More information on how to access the data is here: https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview - The data is downloaded as a NetCDF file. Tutorials for using the Copernicus API are here and work with the NETCDF format are here: https://ecmwf-projects.github.io/copernicus-training-c3s/cds-tutorial.html - """ +The data is downloaded as a NetCDF file. Tutorials for using the Copernicus API are here and work with the NETCDF format are here: https://ecmwf-projects.github.io/copernicus-training-c3s/cds-tutorial.html +""" import gzip import shutil diff --git a/snapshots/climate/2024-06-07/surface_temperature.py b/snapshots/climate/2024-06-07/surface_temperature.py index 8235104ab21..cbc4db8930a 100644 --- a/snapshots/climate/2024-06-07/surface_temperature.py +++ b/snapshots/climate/2024-06-07/surface_temperature.py @@ -1,12 +1,12 @@ """Script to create a snapshot of the monthly averaged surface temperature data from 1950 to present from the Copernicus Climate Change Service. - The script assumes that the data is available on the CDS API. - Instructions on how to access the API on a Mac are here: https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+macOS +The script assumes that the data is available on the CDS API. +Instructions on how to access the API on a Mac are here: https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+macOS - More information on how to access the data is here: https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview +More information on how to access the data is here: https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview - The data is downloaded as a NetCDF file. Tutorials for using the Copernicus API are here and work with the NETCDF format are here: https://ecmwf-projects.github.io/copernicus-training-c3s/cds-tutorial.html - """ +The data is downloaded as a NetCDF file. Tutorials for using the Copernicus API are here and work with the NETCDF format are here: https://ecmwf-projects.github.io/copernicus-training-c3s/cds-tutorial.html +""" import gzip import shutil diff --git a/snapshots/climate/2024-07-08/surface_temperature.py b/snapshots/climate/2024-07-08/surface_temperature.py index 8235104ab21..cbc4db8930a 100644 --- a/snapshots/climate/2024-07-08/surface_temperature.py +++ b/snapshots/climate/2024-07-08/surface_temperature.py @@ -1,12 +1,12 @@ """Script to create a snapshot of the monthly averaged surface temperature data from 1950 to present from the Copernicus Climate Change Service. - The script assumes that the data is available on the CDS API. - Instructions on how to access the API on a Mac are here: https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+macOS +The script assumes that the data is available on the CDS API. +Instructions on how to access the API on a Mac are here: https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+macOS - More information on how to access the data is here: https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview +More information on how to access the data is here: https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview - The data is downloaded as a NetCDF file. Tutorials for using the Copernicus API are here and work with the NETCDF format are here: https://ecmwf-projects.github.io/copernicus-training-c3s/cds-tutorial.html - """ +The data is downloaded as a NetCDF file. Tutorials for using the Copernicus API are here and work with the NETCDF format are here: https://ecmwf-projects.github.io/copernicus-training-c3s/cds-tutorial.html +""" import gzip import shutil diff --git a/snapshots/climate/2024-08-06/surface_temperature.py b/snapshots/climate/2024-08-06/surface_temperature.py index 8235104ab21..cbc4db8930a 100644 --- a/snapshots/climate/2024-08-06/surface_temperature.py +++ b/snapshots/climate/2024-08-06/surface_temperature.py @@ -1,12 +1,12 @@ """Script to create a snapshot of the monthly averaged surface temperature data from 1950 to present from the Copernicus Climate Change Service. - The script assumes that the data is available on the CDS API. - Instructions on how to access the API on a Mac are here: https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+macOS +The script assumes that the data is available on the CDS API. +Instructions on how to access the API on a Mac are here: https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+macOS - More information on how to access the data is here: https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview +More information on how to access the data is here: https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview - The data is downloaded as a NetCDF file. Tutorials for using the Copernicus API are here and work with the NETCDF format are here: https://ecmwf-projects.github.io/copernicus-training-c3s/cds-tutorial.html - """ +The data is downloaded as a NetCDF file. Tutorials for using the Copernicus API are here and work with the NETCDF format are here: https://ecmwf-projects.github.io/copernicus-training-c3s/cds-tutorial.html +""" import gzip import shutil diff --git a/snapshots/climate/latest/weekly_wildfires.csv.dvc b/snapshots/climate/latest/weekly_wildfires.csv.dvc index 7338d1cf709..203d1cf9f10 100644 --- a/snapshots/climate/latest/weekly_wildfires.csv.dvc +++ b/snapshots/climate/latest/weekly_wildfires.csv.dvc @@ -9,12 +9,12 @@ meta: citation_full: Global Wildfire Information System attribution_short: GWIS url_main: https://gwis.jrc.ec.europa.eu/apps/gwis.statistics/seasonaltrend - date_accessed: 2024-11-26 - date_published: 2024-11-26 + date_accessed: 2024-12-12 + date_published: 2024-12-12 license: name: CC BY 4.0 url: https://gwis.jrc.ec.europa.eu/about-gwis/data-license outs: - - md5: 5b4c44cccfe706782c67c1f697f58b62 - size: 12732410 + - md5: 7f65293ec247410a92415d2b3c47a81e + size: 12799314 path: weekly_wildfires.csv diff --git a/snapshots/covid/latest/cases_deaths.csv.dvc b/snapshots/covid/latest/cases_deaths.csv.dvc index c3ed82b1093..15979739510 100644 --- a/snapshots/covid/latest/cases_deaths.csv.dvc +++ b/snapshots/covid/latest/cases_deaths.csv.dvc @@ -12,22 +12,22 @@ meta: All data represent date of reporting as opposed to date of symptom onset. All data are subject to continuous verification and may change based on retrospective updates to accurately reflect trends, changes in country case definitions and/or reporting practices. Significant data errors detected or reported to WHO may be corrected at more frequent intervals. - New case and death counts from the Region of the Americas + **New case and death counts from the Region of the Americas** Starting from the week commencing on 11 September 2023, the source of the data from the Region of the Americas was switched to the aggregated national surveillances, received through the COVID-19, Influenza, RSV and Other Respiratory Viruses program in the Americas. Data have been included retrospectively since 31 July 2023. - Rates + **Rates** <0.001 per 100,000 population may be rounded to 0. citation_full: 'WHO COVID-19 Dashboard. Geneva: World Health Organization, 2020. Available online: https://covid19.who.int/' attribution_short: WHO version_producer: WHO COVID-19 Dashboard - Daily cases and deaths url_main: https://covid19.who.int/ - url_download: https://covid19.who.int/WHO-COVID-19-global-data.csv - date_accessed: 2024-11-26 + url_download: https://srhdpeuwpubsa.blob.core.windows.net/whdh/COVID/WHO-COVID-19-global-daily-data.csv + date_accessed: 2024-12-12 date_published: '2024-07-07' license: name: CC BY 4.0 url: https://data.who.int/dashboards/covid19/ outs: - - md5: 2f146457985eaacb73d38e5cb1aec995 - size: 2860436 + - md5: 16914ffd0a8531ef26e28bc0578eb491 + size: 19539571 path: cases_deaths.csv diff --git a/snapshots/covid/latest/cases_deaths.py b/snapshots/covid/latest/cases_deaths.py index 43b5bb47803..ba96124a716 100644 --- a/snapshots/covid/latest/cases_deaths.py +++ b/snapshots/covid/latest/cases_deaths.py @@ -1,4 +1,13 @@ -"""Script to create a snapshot of dataset.""" +"""Script to create a snapshot of dataset. + +As of 2024-11-29, the WHO reports three files for cases & deaths: + +- [NEW] Daily frequency reporting of new COVID-19 cases and deaths by date reported to WHO: Mostly weekly data, but occasionally daily data (especially past data). +- Weekly COVID-19 cases and deaths by date reported to WHO: Reports weekly values. This is what we have been using since we switched from JHU to WHO. +- Latest reported counts of COVID-19 cases and deaths: Reports latest values (only latest date is available) + + +""" from datetime import date from pathlib import Path diff --git a/snapshots/covid/latest/vaccinations_age.py b/snapshots/covid/latest/vaccinations_age.py index 92eae188976..2baca9db1e4 100644 --- a/snapshots/covid/latest/vaccinations_age.py +++ b/snapshots/covid/latest/vaccinations_age.py @@ -3,6 +3,7 @@ Run the lines from https://github.com/owid/covid-19-data/blob/d8fae5631e2130270532951400fa449a9b20d00b/scripts/src/cowidev/cmd/vax/generate/utils.py#L712-L734, and export df_age. """ + from pathlib import Path from typing import Optional diff --git a/snapshots/covid/latest/vaccinations_global.py b/snapshots/covid/latest/vaccinations_global.py index 40214455d06..b70b648afaf 100644 --- a/snapshots/covid/latest/vaccinations_global.py +++ b/snapshots/covid/latest/vaccinations_global.py @@ -3,6 +3,7 @@ Run the lines from https://github.com/owid/covid-19-data/blob/d8fae5631e2130270532951400fa449a9b20d00b/scripts/src/cowidev/cmd/vax/generate/utils.py#L712-L734, and export df_vaccinations. """ + from datetime import date from pathlib import Path from typing import Optional, cast diff --git a/snapshots/covid/latest/vaccinations_manufacturer.py b/snapshots/covid/latest/vaccinations_manufacturer.py index 96104dfb0fc..17822f5df24 100644 --- a/snapshots/covid/latest/vaccinations_manufacturer.py +++ b/snapshots/covid/latest/vaccinations_manufacturer.py @@ -3,6 +3,7 @@ Run the lines from https://github.com/owid/covid-19-data/blob/d8fae5631e2130270532951400fa449a9b20d00b/scripts/src/cowidev/cmd/vax/generate/utils.py#L712-L734, and export df_manufacturer. """ + from pathlib import Path from typing import Optional diff --git a/snapshots/demography/2024-11-26/multiple_births.7z.dvc b/snapshots/demography/2024-11-26/multiple_births.7z.dvc new file mode 100644 index 00000000000..aec8d447b87 --- /dev/null +++ b/snapshots/demography/2024-11-26/multiple_births.7z.dvc @@ -0,0 +1,63 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Human Multiple Births Database + description: |- + The frequency of twin births has significantly increased in developed countries, doubling since the 1970s (see Figure 1). Two main factors have contributed to this development, namely the rise in the use of medically assisted reproduction techniques, as well as a substantial increase in the mean age at childbearing (Pison et al., 2015). This 'boom' in the birth of twins constitutes a public health challenge, in that twins tend to have frailer health than singletons, at least during their early years. Compared to singletons, twins have lower birth weight, they tend to be born prematurely, and the deliveries are more complicated, all of which can potentially lead to long-term health problems. It is therefore important to understand better the causes of the increase in the twinning rate, as well as the variations across countries. + + The Human Multiple Births Database (HMBD) gathers the number of twin births and the twinning rates for countries with reliable statistics. The database also provides statistics on other multiple births (i.e., triplets, quadruplets, etc.) whenever possible. Although their frequency has increased even more than that of twins, they still constitute a minority, as most multiple deliveries involve twins. A detailed description of the HMBD is available in this article (DOI: 10.4054/DemRes.2023.48.4). + date_published: "2024-09-30" + version_producer: v.1 + + # Citation + producer: Human Multiple Births Database + citation_full: |- + Human Multiple Births Database (2024). French Institute for Demographic Studies - INED (distributor). Extracted from: https://www.twinbirths.org (26/11/2024). + + A detailed description of the HMBD is available in: + Torres, C., Caporali, A., & Pison, G. (2023). The Human Multiple Births Database (HMBD). Demographic Research, 48, 89–106. https://doi.org/10.4054/demres.2023.48.4 + + Country-level sources: + - Australia: Australian Bureau of Statistics (https://www.abs.gov.au/) + - Austria: Statistics Austria (https://www.statistik.at/web_en/statistics/index.html) + - Canada: Statistics Canada (https://www.statcan.gc.ca/) + - Chile: Instituto Nacional de Estadísticas (http://www.ine.cl/) + - Czech Republic: Czech Statistical Office (https://www.czso.cz/csu/czso/home) + - Denmark: Statistics Denmark (https://www.dst.dk/en) + - Finland: Statistics Finland (https://www.stat.fi/index_en.html) + - France: INSEE (https://www.insee.fr/fr/accueil) + - Germany: Statistisches Bundesamt (https://www.destatis.de/EN/Home/_node.html) + - Greece: Hellenic Statistical Authority (https://www.statistics.gr/en/home/) + - Iceland: Statistics Iceland (https://www.statice.is/) + - Italy: ISTAT and Ministerio della Salute (https://www.istat.it/en/, https://www.salute.gov.it/portale/home.html) + - Japan: Ministry of Health, Labour and Welfare (https://www.e-stat.go.jp/en) + - Lithuania: Statistics Lithuania (https://www.stat.gov.lt/en/) + - New Zealand: Statistics New Zealand (https://www.stats.govt.nz/) + - Netherlands: Statistics Netherlands (https://www.cbs.nl/en-gb) + - Norway: Statistics Norway (https://www.ssb.no/en) + - South Korea: Statistics Korea (http://kostat.go.kr/portal/eng/index.action) + - Spain: Instituto Nacional de Estadística (https://www.ine.es/en/index.htm) + - Sweden: Statistics Sweden (https://www.scb.se/en/) + - Switzerland: Federal Statistical Office (https://www.bfs.admin.ch/bfs/en/home.html) + - United States: Centers for Disease Control and Prevention - National Center for Health Statistics (https://www.cdc.gov/nchs/index.htm) + - UK - England and Wales: Office for National Statistics (https://www.ons.gov.uk/) + - UK - Scotland: National Records of Scotland (https://www.nrscotland.gov.uk/) + - Uruguay: Instituto Nacional de Estadística (https://www.gub.uy/ministerio-salud-publica/home) + + attribution_short: HMDB + # Files + url_main: https://www.twinbirths.org/en/data-metadata/ + url_download: https://www.twinbirths.org/fichier/s_rubrique/30699/hmbd_pooled_data_30.09.2024.7z + date_accessed: 2024-11-26 + + # License + license: + name: CC BY 4.0 + url: https://creativecommons.org/licenses/by/4.0/ + +outs: + - md5: 14a9122b39cb033b646c44a915644d63 + size: 69023 + path: multiple_births.7z diff --git a/snapshots/demography/2024-11-26/multiple_births.py b/snapshots/demography/2024-11-26/multiple_births.py new file mode 100644 index 00000000000..51fa8c688d2 --- /dev/null +++ b/snapshots/demography/2024-11-26/multiple_births.py @@ -0,0 +1,24 @@ +"""Script to create a snapshot of dataset.""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +def main(upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"demography/{SNAPSHOT_VERSION}/multiple_births.7z") + + # Download data from source, add file to DVC and upload to S3. + snap.create_snapshot(upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/demography/2024-12-04/paternal_ages.py b/snapshots/demography/2024-12-04/paternal_ages.py new file mode 100644 index 00000000000..1173367aab3 --- /dev/null +++ b/snapshots/demography/2024-12-04/paternal_ages.py @@ -0,0 +1,28 @@ +"""Script to create a snapshot of dataset. + +The data for this snapshot was obtained via private communication with the authors of the original study +""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +@click.option("--path-to-file", "-f", prompt=True, type=str, help="Path to local data file.") +def main(path_to_file: str, upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"demography/{SNAPSHOT_VERSION}/paternal_ages.rdata") + + # Copy local data file to snapshots data folder, add file to DVC and upload to S3. + snap.create_snapshot(filename=path_to_file, upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/demography/2024-12-04/paternal_ages.rdata.dvc b/snapshots/demography/2024-12-04/paternal_ages.rdata.dvc new file mode 100644 index 00000000000..0c7a28f91ad --- /dev/null +++ b/snapshots/demography/2024-12-04/paternal_ages.rdata.dvc @@ -0,0 +1,26 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: LPAC Database, Long-term Trends in Paternal Age at Childbirth + description: |- + The research article investigates long-term trends in paternal age at childbirth, revealing a U-shaped pattern across various countries. While recent increases in paternal age have raised concerns about offspring health, the study demonstrates that current levels are not unprecedented. The authors highlight the need for further research to understand the interplay between biological and social factors influencing offspring health outcomes at both individual and population levels, including the impact of medical advancements and changing social norms. A significant knowledge gap exists regarding the interaction of maternal and paternal age effects. Finally, a product advertisement promotes a medium to improve pregnancy chances by supporting embryo-endometrial communication. + date_published: "2024-04-03" + # Citation + producer: Willführ and Klüsener + citation_full: |- + Willführ, K. P., Fox, J. F., & Voland, E. (2024). 26. Historical Family Reconstitution Databases in the Study of Kinship Influences on Demographic Outcomes. In Open Book Publishers (pp. 617–634). https://doi.org/10.11647/obp.0251.26 + + # Files + url_main: https://lpac-database.shinyapps.io/Data/ + date_accessed: 2024-12-04 + + # License + license: + name: CC BY 4.0 + url: https://creativecommons.org/licenses/by/4.0/deed.en +outs: + - md5: a3fc2cd51a9a0e638064b47b4fb99cb6 + size: 928662 + path: paternal_ages.rdata diff --git a/snapshots/demography/2024-12-06/wittgenstein_human_capital.py b/snapshots/demography/2024-12-06/wittgenstein_human_capital.py new file mode 100644 index 00000000000..da013c6fd64 --- /dev/null +++ b/snapshots/demography/2024-12-06/wittgenstein_human_capital.py @@ -0,0 +1,202 @@ +"""Script to create a snapshot of dataset.""" + +import os +import shutil +from pathlib import Path +from typing import Optional + +import click +import requests + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + +# Latest version available +URL_SERVER = "https://wicshiny2023.iiasa.ac.at/wcde-data/wcde-v3-batch" +INDICATORS = [ + "asfr", + "assr", + "bmys", + "bpop", + "bprop", + "cbr", + "cdr", + "e0", + "easfr", + "eassr", + "emacb", + "emi", + "epop", + "etfr", + "ggapedu15", + "ggapedu25", + "ggapmys15", + "ggapmys25", + "growth", + "imm", + "macb", + "mage", + "mys", + "net", + "netedu", + "nirate", + "odr", + "pop-age-edattain", + "pop-age-sex-edattain", + "pop-age-sex", + "pop-age", + "pop-sex-edattain", + "pop-sex", + "pop-total", + "pop", + "prop", + "pryl15", + "ryl15", + "sexratio", + "tdr", + "tfr", + "ydr", +] +SCENARIOS = [ + "1", + "2", + "22", + "23", + "3", + "4", + "5", +] + +# Batch with historical data +URL_SERVER_HISTORICAL = "https://wicshiny2023.iiasa.ac.at/wcde-data/data-batch/" +SCENARIOS_HISTORICAL = [ + "ssp1", + "ssp2", + "ssp3", + "ssp4", + "ssp5", +] +INDICATORS_HISTORICAL = [ + "asfr", + "assr", + "bmys", + "bpop", + "bprop", + "cbr", + "cdr", + "e0", + "easfr", + "eassr", + # "emacb", + # "emi", + "epop", + "etfr", + "ggapedu15", + "ggapedu25", + "ggapmys15", + "ggapmys25", + "growth", + # "imm", + "macb", + "mage", + "mys", + "net", + # "netedu", + "nirate", + "odr", + # "pop-age-edattain", + # "pop-age-sex-edattain", + # "pop-age-sex", + # "pop-age", + # "pop-sex-edattain", + # "pop-sex", + # "pop-total", + "pop", + "prop", + "pryl15", + "ryl15", + "sexratio", + "tdr", + "tfr", + "ydr", +] + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +@click.option("--path-to-file", "-f1", type=str, help="Path to local data file.") +@click.option("--path-to-file-historical", "-f2", type=str, help="Path to local data file.") +def main(path_to_file: Optional[str], path_to_file_historical: str, upload: bool) -> None: + if path_to_file is not None: + # Create a new snapshot. + snap = Snapshot(f"demography/{SNAPSHOT_VERSION}/wittgenstein_human_capital.zip") + + # Copy local data file to snapshots data folder, add file to DVC and upload to S3. + snap.create_snapshot(filename=path_to_file, upload=upload) + + if path_to_file_historical is not None: + # Create a new snapshot. + snap = Snapshot(f"demography/{SNAPSHOT_VERSION}/wittgenstein_human_capital_historical.zip") + + # Copy local data file to snapshots data folder, add file to DVC and upload to S3. + snap.create_snapshot(filename=path_to_file_historical, upload=upload) + + +def get_data(folder: str): + """Download all files from the WCDE server. + + Relevant files: + - All scenarios in SCENARIOS + - All indicators in INDICATORS + + Saves all these files in `folder`. + """ + os.makedirs(folder, exist_ok=True) + for scenario in SCENARIOS: + for indicator in INDICATORS: + url = f"{URL_SERVER}/{scenario}/{indicator}.rds" + try: + file_path = f"{folder}/{scenario}_{indicator}.rds" + response = requests.get(url) + response.raise_for_status() # Raise HTTPError for bad responses + with open(file_path, "wb") as file: + file.write(response.content) + print(f"Saved: {file_path}") + except requests.exceptions.RequestException as e: + print(f"Failed to download {url}: {e}") + + # Zip file + shutil.make_archive(folder, "zip", folder) + + +def get_data_historical(folder: str): + """Download all files from the WCDE server (historical values). + + Relevant files: + - All scenarios in SCENARIOS_HISTORICAL + - All indicators in INDICATORS + + Saves all these files in `folder`. + """ + os.makedirs(folder, exist_ok=True) + for scenario in SCENARIOS_HISTORICAL: + for indicator in INDICATORS_HISTORICAL: + url = f"{URL_SERVER_HISTORICAL}/{scenario}/{indicator}.csv.gz" + try: + file_path = f"{folder}/{scenario}_{indicator}.csv.gz" + response = requests.get(url) + response.raise_for_status() # Raise HTTPError for bad responses + with open(file_path, "wb") as file: + file.write(response.content) + print(f"Saved: {file_path}") + except requests.exceptions.RequestException as e: + print(f"Failed to download {url}: {e}") + + # Zip file + shutil.make_archive(folder, "zip", folder) + + +if __name__ == "__main__": + main() diff --git a/snapshots/demography/2024-12-06/wittgenstein_human_capital.zip.dvc b/snapshots/demography/2024-12-06/wittgenstein_human_capital.zip.dvc new file mode 100644 index 00000000000..ad14a25bd53 --- /dev/null +++ b/snapshots/demography/2024-12-06/wittgenstein_human_capital.zip.dvc @@ -0,0 +1,32 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Human Capital, Wittgenstein Centre + description: |- + This website presents the 2023 revision of the Wittgenstein Centre population projections (Version 3), following the initial projections (Version 1) in 2013 (Lutz, Butz, and KC (Eds.) 2014) and the 2018 update (Version 2) (Lutz, Goujon, KC, Stonawski, and, Stilianakis (Eds.) 2018). Further details on the updates and the revisions can be found in K.C. et al. (2024) and in other satellite papers focusing on the migration component (Yildiz and Abel 2024), the mortality component (Dhakad and KC 2024 - forthcoming), and the fertility component (Adhikari et al. 2024 - forthcoming). + + In 2021, the Shared Socioeconomic Pathways (SSPs) scenario community related to the Intergovernmental Panel on Climate Change requested an update of the human core of the SSPs, which is at the origin of this version. This updated version is based on 2020 as the reference year, with adjustments to certain short-term assumptions extending to 2030. However, most long-term assumptions have not been changed. Several changes were nevertheless implemented particularly in estimating and using country-specific education differentials in mortality and migration. + + This new version includes population projections by levels of educational attainment from 2020 to 2100 for 200 countries according to seven scenarios (SSP1-5, SSP2-Zero Migration and SSP2-Double Migration). The reconstruction will become available in the course of 2024. + version_producer: 3 + date_published: "2024-02-08" + + # Citation + producer: Wittgenstein Centre + citation_full: |- + K.C., S., Dhakad, M., Potancokova, M., Adhikari, S., Yildiz, D., Mamolo, M., Sobotka, T., Zeman, K., Abel, G., Lutz, W., and Goujon, A. (2024). Updating the Shared Socioeconomic Pathways (SSPs) Global Population and Human Capital Projections. IIASA Working Paper. Laxenburg, Austria: WP-24-003. https://pure.iiasa.ac.at/19487. Available at: http://www.wittgensteincentre.org/dataexplorer. + + # Files + url_main: https://dataexplorer.wittgensteincentre.org/wcde-v3/ + date_accessed: 2024-12-06 + + # License + license: + name: CC BY 4.0 + url: https://creativecommons.org/licenses/by/4.0/deed.en +outs: + - md5: a1bc7bb756f24d5a0e6eb1fe90f057e5 + size: 115287227 + path: wittgenstein_human_capital.zip diff --git a/snapshots/demography/2024-12-06/wittgenstein_human_capital_historical.zip.dvc b/snapshots/demography/2024-12-06/wittgenstein_human_capital_historical.zip.dvc new file mode 100644 index 00000000000..f84a0d74b6b --- /dev/null +++ b/snapshots/demography/2024-12-06/wittgenstein_human_capital_historical.zip.dvc @@ -0,0 +1,32 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Human Capital, Wittgenstein Centre + description: |- + This website presents the 2023 revision of the Wittgenstein Centre population projections (Version 3), following the initial projections (Version 1) in 2013 (Lutz, Butz, and KC (Eds.) 2014) and the 2018 update (Version 2) (Lutz, Goujon, KC, Stonawski, and, Stilianakis (Eds.) 2018). Further details on the updates and the revisions can be found in K.C. et al. (2024) and in other satellite papers focusing on the migration component (Yildiz and Abel 2024), the mortality component (Dhakad and KC 2024 - forthcoming), and the fertility component (Adhikari et al. 2024 - forthcoming). + + In 2021, the Shared Socioeconomic Pathways (SSPs) scenario community related to the Intergovernmental Panel on Climate Change requested an update of the human core of the SSPs, which is at the origin of this version. This updated version is based on 2020 as the reference year, with adjustments to certain short-term assumptions extending to 2030. However, most long-term assumptions have not been changed. Several changes were nevertheless implemented particularly in estimating and using country-specific education differentials in mortality and migration. + + This new version includes population projections by levels of educational attainment from 2020 to 2100 for 200 countries according to seven scenarios (SSP1-5, SSP2-Zero Migration and SSP2-Double Migration). The reconstruction will become available in the course of 2024. + title_snapshot: Human Capital, Wittgenstein Centre (Historical data) + date_published: "2024-02-08" + + # Citation + producer: Wittgenstein Centre + citation_full: |- + K.C., S., Dhakad, M., Potancokova, M., Adhikari, S., Yildiz, D., Mamolo, M., Sobotka, T., Zeman, K., Abel, G., Lutz, W., and Goujon, A. (2024). Updating the Shared Socioeconomic Pathways (SSPs) Global Population and Human Capital Projections. IIASA Working Paper. Laxenburg, Austria: WP-24-003. https://pure.iiasa.ac.at/19487. Available at: http://www.wittgensteincentre.org/dataexplorer. + + # Files + url_main: https://dataexplorer.wittgensteincentre.org/wcde-v3/ + date_accessed: 2024-12-06 + + # License + license: + name: CC BY 4.0 + url: https://creativecommons.org/licenses/by/4.0/deed.en +outs: + - md5: 6aaf6017779daed3356935b745d8d25e + size: 318304437 + path: wittgenstein_human_capital_historical.zip diff --git a/snapshots/ember/2024-11-20/european_wholesale_electricity_prices.csv.dvc b/snapshots/ember/2024-11-20/european_wholesale_electricity_prices.csv.dvc new file mode 100644 index 00000000000..ba348bf3808 --- /dev/null +++ b/snapshots/ember/2024-11-20/european_wholesale_electricity_prices.csv.dvc @@ -0,0 +1,30 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: European Wholesale Electricity Price Data + description: |- + Wholesale day-ahead electricity price data for European countries. + date_published: "2024-11-20" + title_snapshot: European Wholesale Electricity Price Data - Monthly + + # Citation + producer: Ember + citation_full: |- + Ember - European Wholesale Electricity Price Data. Based on data from European Network of Transmission System Operators (ENTSO-E). + + # Files + url_main: https://ember-energy.org/data/european-wholesale-electricity-price-data/ + url_download: https://storage.googleapis.com/emb-prod-bkt-publicdata/public-downloads/european_wholesale_electricity_price_data_monthly.csv + date_accessed: 2024-11-20 + + # License + license: + name: CC BY 4.0 + url: https://ember-energy.org/creative-commons/ + +outs: + - md5: ce52ff862b464953c93b1f04531d0db5 + size: 94675 + path: european_wholesale_electricity_prices.csv diff --git a/snapshots/ember/2024-11-20/european_wholesale_electricity_prices.py b/snapshots/ember/2024-11-20/european_wholesale_electricity_prices.py new file mode 100644 index 00000000000..2f52db1b75d --- /dev/null +++ b/snapshots/ember/2024-11-20/european_wholesale_electricity_prices.py @@ -0,0 +1,24 @@ +"""Script to create a snapshot of dataset.""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +def main(upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"ember/{SNAPSHOT_VERSION}/european_wholesale_electricity_prices.csv") + + # Download data from source, add file to DVC and upload to S3. + snap.create_snapshot(upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/ess/2023-08-02/ess_trust.py b/snapshots/ess/2023-08-02/ess_trust.py index 0ded6074a04..b5b44ac48d5 100644 --- a/snapshots/ess/2023-08-02/ess_trust.py +++ b/snapshots/ess/2023-08-02/ess_trust.py @@ -5,14 +5,14 @@ INSTRUCTIONS - 1. In the ESS Data Portal, register and download all the survey files from the Data Wizard, https://ess-search.nsd.no/CDW/RoundCountry. - 2. Extract the dta file from the zip file. - 3. Run this ess_trust.do in Stata. If it fails, check the name of the dta file in the first line of the code. - 4. The output is given in Stata's output window. Copy and paste it into a csv file, called `ess_trust.csv`. - 5. Add snapshot. The command is: - python snapshots/ess/{version}/ess_trust.py --path-to-file snapshots/ess/{version}/ess_trust.csv - 6. Delete csv file - 7. Run `etl ess_trust` + 1. In the ESS Data Portal, register and download all the survey files from the Data Wizard, https://ess-search.nsd.no/CDW/RoundCountry. + 2. Extract the dta file from the zip file. + 3. Run this ess_trust.do in Stata. If it fails, check the name of the dta file in the first line of the code. + 4. The output is given in Stata's output window. Copy and paste it into a csv file, called `ess_trust.csv`. + 5. Add snapshot. The command is: + python snapshots/ess/{version}/ess_trust.py --path-to-file snapshots/ess/{version}/ess_trust.csv + 6. Delete csv file + 7. Run `etl ess_trust` """ diff --git a/snapshots/eurostat/2024-11-05/gas_and_electricity_prices.py b/snapshots/eurostat/2024-11-05/gas_and_electricity_prices.py new file mode 100644 index 00000000000..b0bf9e6d6af --- /dev/null +++ b/snapshots/eurostat/2024-11-05/gas_and_electricity_prices.py @@ -0,0 +1,90 @@ +"""Script to create a snapshot of dataset.""" + +import zipfile +from pathlib import Path + +import click +import requests +from tqdm.auto import tqdm + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + +# Base URL for Eurostat API energy data. +BASE_URL = "https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/data/" + +# List of dataset codes to download. +URL_DOWNLOADS = [ + #################################################################################################################### + # Energy statistics - natural gas and electricity prices (from 2007 onwards) (nrg_pc) + # Gas prices for household consumers - bi-annual data (from 2007 onwards) (nrg_pc_202) + "nrg_pc_202", + # Gas prices for non-household consumers - bi-annual data (from 2007 onwards) (nrg_pc_203) + "nrg_pc_203", + # Electricity prices for household consumers - bi-annual data (from 2007 onwards) (nrg_pc_204) + "nrg_pc_204", + # Electricity prices for non-household consumers - bi-annual data (from 2007 onwards) (nrg_pc_205) + "nrg_pc_205", + # Household consumption volumes of gas by consumption bands (nrg_pc_202_v) + "nrg_pc_202_v", + # Non-household consumption volumes of gas by consumption bands (nrg_pc_203_v) + "nrg_pc_203_v", + # Household consumption volumes of electricity by consumption bands (nrg_pc_204_v) + "nrg_pc_204_v", + # Non-household consumption volumes of electricity by consumption bands (nrg_pc_205_v) + "nrg_pc_205_v", + # Gas prices components for household consumers - annual data (nrg_pc_202_c) + "nrg_pc_202_c", + # Gas prices components for non-household consumers - annual data (nrg_pc_203_c) + "nrg_pc_203_c", + # Electricity prices components for household consumers - annual data (from 2007 onwards) (nrg_pc_204_c) + "nrg_pc_204_c", + # Electricity prices components for non-household consumers - annual data (from 2007 onwards) (nrg_pc_205_c) + "nrg_pc_205_c", + # Share for transmission and distribution in the network cost for gas and electricity - annual data (nrg_pc_206) + "nrg_pc_206", + #################################################################################################################### + # Energy statistics - natural gas and electricity prices (until 2007) (nrg_pc_h) + # Gas prices for domestic consumers - bi-annual data (until 2007) (nrg_pc_202_h) + "nrg_pc_202_h", + # Gas prices for industrial consumers - bi-annual data (until 2007) (nrg_pc_203_h) + "nrg_pc_203_h", + # Electricity prices for domestic consumers - bi-annual data (until 2007) (nrg_pc_204_h) + "nrg_pc_204_h", + # Electricity prices for industrial consumers - bi-annual data (until 2007) (nrg_pc_205_h) + "nrg_pc_205_h", + # Electricity - marker prices - bi-annual data (until 2007) (nrg_pc_206_h) + "nrg_pc_206_h", + #################################################################################################################### +] +# Further API parameters to download each file. +URL_SUFFIX = "?format=TSV&compressed=false" + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +def main(upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"eurostat/{SNAPSHOT_VERSION}/gas_and_electricity_prices.zip") + + # Ensure output snapshot folder exists, otherwise create it. + snap.path.parent.mkdir(exist_ok=True, parents=True) + + # Create a temporary ZIP file. + with zipfile.ZipFile(snap.path, "w") as zip_file: + # Fetch all relevant datasets from Eurostat API. + for code in tqdm(URL_DOWNLOADS): + # Request the data file for the current dataset. + response = requests.get(f"{BASE_URL}{code}{URL_SUFFIX}") + # Save each file inside the ZIP file. + file_name = f"{code}.tsv" + zip_file.writestr(file_name, response.text) + + # Create snapshot and upload to R2. + snap.create_snapshot(upload=upload, filename=snap.path) + + +if __name__ == "__main__": + main() diff --git a/snapshots/eurostat/2024-11-05/gas_and_electricity_prices.zip.dvc b/snapshots/eurostat/2024-11-05/gas_and_electricity_prices.zip.dvc new file mode 100644 index 00000000000..96d2eb2963a --- /dev/null +++ b/snapshots/eurostat/2024-11-05/gas_and_electricity_prices.zip.dvc @@ -0,0 +1,26 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Energy statistics, prices of natural gas and electricity + date_published: "2024-11-04" + + # Citation + producer: Eurostat + citation_full: |- + Eurostat - Energy statistics, prices of natural gas and electricity (2024). + + # Files + url_main: https://ec.europa.eu/eurostat/web/energy/database + date_accessed: 2024-11-05 + + # License + license: + name: CC BY 4.0 + url: https://ec.europa.eu/eurostat/web/main/help/copyright-notice + +outs: + - md5: 285787159fb3f598db8c0fb0ba67eb2c + size: 8286547 + path: gas_and_electricity_prices.zip diff --git a/snapshots/excess_mortality/latest/hmd_stmf.csv.dvc b/snapshots/excess_mortality/latest/hmd_stmf.csv.dvc index 1a5cba23ad7..c5b668bcb11 100644 --- a/snapshots/excess_mortality/latest/hmd_stmf.csv.dvc +++ b/snapshots/excess_mortality/latest/hmd_stmf.csv.dvc @@ -13,7 +13,7 @@ meta: HMD provides an online STMF visualization toolkit (https://mpidr.shinyapps.io/stmortality). url: https://www.mortality.org/Data/STMF source_data_url: https://www.mortality.org/File/GetDocument/Public/STMF/Outputs/stmf.csv - date_accessed: 2024-11-26 + date_accessed: 2024-12-12 publication_date: 2024-11-11 publication_year: 2024 published_by: |- diff --git a/snapshots/excess_mortality/latest/hmd_stmf.py b/snapshots/excess_mortality/latest/hmd_stmf.py index c573fa05c30..c102ec5f2b9 100644 --- a/snapshots/excess_mortality/latest/hmd_stmf.py +++ b/snapshots/excess_mortality/latest/hmd_stmf.py @@ -1,4 +1,5 @@ """Script to create a snapshot of dataset 'Short-Term Mortality Fluctuations (HMD, 2023)'.""" + import re from datetime import date, datetime from pathlib import Path diff --git a/snapshots/excess_mortality/latest/wmd.csv.dvc b/snapshots/excess_mortality/latest/wmd.csv.dvc index 5cdf9e772ea..3ba02875ab2 100644 --- a/snapshots/excess_mortality/latest/wmd.csv.dvc +++ b/snapshots/excess_mortality/latest/wmd.csv.dvc @@ -13,7 +13,7 @@ meta: Published paper available at https://elifesciences.org/articles/69336. url: https://github.com/akarlinsky/world_mortality/ source_data_url: https://raw.githubusercontent.com/akarlinsky/world_mortality/main/world_mortality.csv - date_accessed: 2024-11-26 + date_accessed: 2024-12-12 publication_date: '2021-06-30' publication_year: 2021 published_by: |- @@ -33,6 +33,6 @@ meta: name: MIT License url: https://github.com/akarlinsky/world_mortality/blob/main/LICENSE outs: - - md5: 2df7e56f5f496e18a07780bf7dd0af07 - size: 1084531 + - md5: cab03dff0de45a45aae54fe9772c4666 + size: 1087717 path: wmd.csv diff --git a/snapshots/excess_mortality/latest/xm_karlinsky_kobak.csv.dvc b/snapshots/excess_mortality/latest/xm_karlinsky_kobak.csv.dvc index 7572a4d49c7..099a3bf87f7 100644 --- a/snapshots/excess_mortality/latest/xm_karlinsky_kobak.csv.dvc +++ b/snapshots/excess_mortality/latest/xm_karlinsky_kobak.csv.dvc @@ -7,7 +7,7 @@ meta: For more details, refer to https://github.com/dkobak/excess-mortality#excess-mortality-during-the-covid-19-pandemic. url: https://github.com/dkobak/excess-mortality source_data_url: https://raw.githubusercontent.com/dkobak/excess-mortality/main/baselines-per-year.csv - date_accessed: 2024-11-26 + date_accessed: 2024-12-12 publication_date: '2021-06-30' publication_year: 2021 published_by: |- diff --git a/snapshots/excess_mortality/latest/xm_karlinsky_kobak.py b/snapshots/excess_mortality/latest/xm_karlinsky_kobak.py index d484e22bf02..6d175af88dd 100644 --- a/snapshots/excess_mortality/latest/xm_karlinsky_kobak.py +++ b/snapshots/excess_mortality/latest/xm_karlinsky_kobak.py @@ -1,4 +1,5 @@ """Script to create a snapshot of dataset 'Excess mortality during the COVID-19 pandemic'.""" + from datetime import date from pathlib import Path diff --git a/snapshots/excess_mortality/latest/xm_karlinsky_kobak_ages.csv.dvc b/snapshots/excess_mortality/latest/xm_karlinsky_kobak_ages.csv.dvc index b4e5f99b970..872549d2d0c 100644 --- a/snapshots/excess_mortality/latest/xm_karlinsky_kobak_ages.csv.dvc +++ b/snapshots/excess_mortality/latest/xm_karlinsky_kobak_ages.csv.dvc @@ -6,7 +6,7 @@ meta: For more details, refer to https://github.com/dkobak/excess-mortality#excess-mortality-during-the-covid-19-pandemic. url: https://github.com/dkobak/excess-mortality source_data_url: https://raw.githubusercontent.com/dkobak/excess-mortality/main/baselines-stmf.csv - date_accessed: 2024-11-26 + date_accessed: 2024-12-12 publication_date: '2021-06-30' publication_year: 2021 published_by: |- diff --git a/snapshots/fasttrack/2022-11-01/lighting_efficiency_uk.csv.dvc b/snapshots/fasttrack/2022-11-01/lighting_efficiency_uk.csv.dvc index a8571c72af8..40805443ffa 100644 --- a/snapshots/fasttrack/2022-11-01/lighting_efficiency_uk.csv.dvc +++ b/snapshots/fasttrack/2022-11-01/lighting_efficiency_uk.csv.dvc @@ -1,18 +1,18 @@ meta: - namespace: fasttrack - short_name: lighting_efficiency_uk - file_extension: csv - date_accessed: 2023-01-20 + source: + name: |- + Fouquet & Pearson (2006). Seven centuries of energy services: The price and use of light in the United Kingdom (1300-2000). + url: https://www.jstor.org/stable/23296980 + source_data_url: |- + gAAAAABnRYZlXxepMAB9sIKn_qv_qmZn_zxaLMC7r6W0Pm4yDGaffkjMBQa1ovpzqTVnmQFcclmueGVS_TPk9_ethfFaKeAVFo7yRlvM-T5Vn56IJP1ZDGZzGF1pZQtvwvsWCYq7O_ZzRq9mGI439QHZ5s-0oewSvJFcpvI_Nc1EqVdsO-Hyb3hKhvCdXuR5Sg_E0gtagMAeh0KT8kAYTIxfAVtj29fY5yVUXXQU76CEkSRtenAVfSYjMD7hwik8ZFaG-fS-XRrP + date_accessed: '2024-11-26' + published_by: |- + Fouquet, R., & Pearson, P. J. (2006). Seven centuries of energy services: The price and use of light in the United Kingdom (1300-2000). The energy journal, 27(1). name: Lighting effiency and shares in the UK description: '' - source_name: Google Sheet - url: https://www.jstor.org/stable/23296980 - source_published_by: Google Sheet - source_data_url: gAAAAABjykepDM9qSPmcia30Y6XfRfCAr-IVVU4MKQ-5uronVcInRAbQENcbYoFR7vrbbSMRsNqulOO1uRMVdAZ3lQIdTl_fQNWcRFmxC1V8nujs-6pa5aXPNDAmm4mOtKD_d--xITDJEdrwIFi-xCKtZmx7oLm8b-sbG2b1WEvUHKbJAIARIgw1PEA_7KQEhiKmbnpU-SjalCi-9PAeMKUDtsQhxAYSBtkOUwXt7_HCq5UBYPvqA_N8fUw5pb7MDILs0bScwGzY + license: {} is_public: false - version: '2022-11-01' -wdir: ../../../data/snapshots/fasttrack/2022-11-01 outs: -- md5: 7e2d4e77b9c23946c89c518df591a009 - size: 747 - path: lighting_efficiency_uk.csv + - md5: 8de6c6871078ece444a6c927db29ad48 + size: 679 + path: lighting_efficiency_uk.csv diff --git a/snapshots/fasttrack/2023-01-19/food_expenditures_by_country.csv.dvc b/snapshots/fasttrack/2023-01-19/food_expenditures_by_country.csv.dvc deleted file mode 100644 index e0179559e5b..00000000000 --- a/snapshots/fasttrack/2023-01-19/food_expenditures_by_country.csv.dvc +++ /dev/null @@ -1,18 +0,0 @@ -meta: - namespace: fasttrack - short_name: food_expenditures_by_country - file_extension: csv - date_accessed: 2023-01-20 - name: Food expenditures by country (USDA, 2023) - description: '' - source_name: Google Sheet - url: https://www.ers.usda.gov/topics/international-markets-u-s-trade/international-consumer-and-food-industry-trends/#data - source_published_by: Google Sheet - source_data_url: gAAAAABjyoIXcZhrrY4oOnHOyLS8RohUA72-X3Z1ZHFU7A_Vmg60iZNiXhYuAi99P34Qs5wXDznB_BK-KyzpkEgsV9mGZ5miuZGV0RPrTrIrHdqBDozjzbbYdUE5N_yzDQFtHTcsD_2cIiqrqby4ZJrM2xRnegMQ5kQvMD8tdajOp9438_qAgKpz5ejRVYabGGuL88EXxtOds8UsvHhaPqwMb9_xe7GXVdEGRNIsEzzSJqZHFkCkj_R-Vzg2lHP1tgSLVeULyb2i - is_public: false - version: '2023-01-19' -wdir: ../../../data/snapshots/fasttrack/2023-01-19 -outs: -- md5: b03947c40e7c56fa98f08b06fea563e4 - size: 6497 - path: food_expenditures_by_country.csv diff --git a/snapshots/fasttrack/2023-03-27/global_warming_contributions.csv.dvc b/snapshots/fasttrack/2023-03-27/global_warming_contributions.csv.dvc index 596a41eeb39..574f274fa94 100644 --- a/snapshots/fasttrack/2023-03-27/global_warming_contributions.csv.dvc +++ b/snapshots/fasttrack/2023-03-27/global_warming_contributions.csv.dvc @@ -1,8 +1,13 @@ meta: - namespace: fasttrack - short_name: global_warming_contributions - file_extension: csv - date_accessed: 2023-03-29 + source: + name: Matthew Jones et al. (2023) + url: https://doi.org/10.5281/zenodo.7636699 + source_data_url: |- + gAAAAABnRYaj4DTXeJP0ir8ExcuaJRXjM2guld5knTOnakD5MVRDT9gg0A5PS-3i584pjGjV-RpP-30PBWfwi7DcqsY2qcDyjtdELFAz_0YeiJPZBPjb6SgpO5xGPJq5HBzJugTrPm15GuJ0ntAZGz3mh13Ev2eNs4AdpXcM5R2V-z9Ru3PdHs-ZFIpHVXUOD3zw5cGmm5mlCNybYfiqvFEPuhSgCP5-aWk2wZdj9Nv6rFUK6g2F1IKygoH1DUS5Gk4TciIBKcaT + date_accessed: '2024-11-26' + publication_year: 2023 + published_by: |- + Jones, Matthew W., Peters, Glen P., Gasser, Thomas, Andrew, Robbie M., Schwingshackl, Clemens, Gütschow, Johannes, Houghton, Richard A., Friedlingstein, Pierre, Pongratz, Julia, & Le Quéré, Corinne. (2023). National contributions to climate change due to historical emissions of carbon dioxide, methane and nitrous oxide [Data set]. In Scientific Data (2023.1). name: Global warming contributions description: |- Jones et al. (2023) quantify national and regional contributions to the increase of global mean surface temperature over the last few centuries. As they detail: the "dataset describing the global warming response to national emissions CO2, CH4 and N2O from fossil and land use sources during 1851-2021. @@ -14,15 +19,9 @@ meta: We construct a time series of cumulative CO2-equivalent emissions for each country, gas, and emissions source (fossil or land use). Emissions of CH4 and N2O emissions are related to cumulative CO2-equivalent emissions using the Global Warming Potential (GWP*) approach, with best-estimates of the coefficients taken from the IPCC AR6 (Forster et al., 2021). Warming in response to cumulative CO2-equivalent emissions is estimated using the transient climate response to cumulative carbon emissions (TCRE) approach, with best-estimate value of TCRE taken from the IPCC AR6 (Forster et al., 2021, Canadell et al., 2021). 'Warming' is specifically the change in global mean surface temperature (GMST)." - source_name: Google Sheet - url: https://doi.org/10.5281/zenodo.7636699 - source_published_by: Google Sheet - source_data_url: gAAAAABkJA7WvCo-f3u_JMGUF9cid_2fGu1DIt-m-5KWm8yczckrjMqCtatgoUgl0ndutC89gy_T0x7nP0SLoOLFwT9QPPBnHhk1q4NVdXRQmQGqtMUZ9fpI3ihqHUGi536Kh2baRyUgXWQFBG_3QPLq0gLEMs2o9zYNx_e0ZxsXrNrIsTCjtopQEeV4Wwc_h2dk_5l4i-MYFbNSbqs6hXMMmUAKRMIKmwxQG8pKnjgAoYSQZclY581EqJrFjRKnmFGSXbLnc6i4 + license: {} is_public: false - version: '2023-03-27' - publication_year: 2023 -wdir: ../../../data/snapshots/fasttrack/2023-03-27 outs: -- md5: 48f8ff1455c82c193d7e5ec1cbc58fb7 - size: 27352636 - path: global_warming_contributions.csv + - md5: 237c2f9f1ec5ffc19978b1f40d36b028 + size: 19377131 + path: global_warming_contributions.csv diff --git a/snapshots/fasttrack/2023-04-30/paratz.csv.dvc b/snapshots/fasttrack/2023-04-30/paratz.csv.dvc index d6cef55f5b9..cdab99d5e09 100644 --- a/snapshots/fasttrack/2023-04-30/paratz.csv.dvc +++ b/snapshots/fasttrack/2023-04-30/paratz.csv.dvc @@ -1,34 +1,19 @@ meta: - name: A systematic review of global autopsy rates in all-cause mortality and young - sudden death, Paratz et al (2023) - description: >- - The data for this indicator is taken from: Paratz ED, Rowe SJ, Stub D, Pflaumer - A, La Gerche A. A systematic review of - global autopsy rates in all-cause mortality and young sudden death. Heart Rhythm. - 2023 Apr;20(4):607-613. doi: 10.1016/j.hrthm.2023.01.008. - - - The data is collated from a number of published papers and databases. The year - shown reflects the date given in the database - or the year of the publication. For Spain and Australia the data is only representative - of a region of each country, Catalonia - and Victoria, respectively. source: - name: Google Sheet - description: + name: Paratz et al., (2023) url: https://www.heartrhythmjournal.com/article/S1547-5271(23)00027-9/fulltext - source_data_url: + source_data_url: |- https://docs.google.com/spreadsheets/d/e/2PACX-1vROXfyuCq1_aNHxIkLGOhbi9xZmVHFvPltm44VU__skPYh4Gn9ES8oLcgI8okIF7D4Sts_gjD9568_O/pub?output=csv - owid_data_url: - date_accessed: '2023-07-13' - publication_date: + date_accessed: '2024-11-26' publication_year: 2023 - published_by: Google Sheet - license: - name: - url: -wdir: ../../../data/snapshots/fasttrack/2023-04-30 + published_by: Heart Rhythm Journal + name: A systematic review of global autopsy rates in all-cause mortality and young sudden death, Paratz et al (2023) + description: |- + The data for this indicator is taken from: Paratz ED, Rowe SJ, Stub D, Pflaumer A, La Gerche A. A systematic review of global autopsy rates in all-cause mortality and young sudden death. Heart Rhythm. 2023 Apr;20(4):607-613. doi: 10.1016/j.hrthm.2023.01.008. + + The data is collated from a number of published papers and databases. The year shown reflects the date given in the database or the year of the publication. For Spain and Australia the data is only representative of a region of each country, Catalonia and Victoria, respectively. + license: {} outs: -- md5: 14f4576ea63fda9388db9f2c3f596990 - size: 1110 - path: paratz.csv + - md5: 14f4576ea63fda9388db9f2c3f596990 + size: 1110 + path: paratz.csv diff --git a/snapshots/fasttrack/2023-05-03/apms_2014.csv.dvc b/snapshots/fasttrack/2023-05-03/apms_2014.csv.dvc index f73e679bdc0..e9c64b5d942 100644 --- a/snapshots/fasttrack/2023-05-03/apms_2014.csv.dvc +++ b/snapshots/fasttrack/2023-05-03/apms_2014.csv.dvc @@ -1,26 +1,21 @@ meta: - namespace: fasttrack - short_name: apms_2014 - file_extension: csv - date_accessed: 2023-05-03 + source: + name: Adult Psychiatric Morbidity Survey 2014, England (2016) + url: https://www.gov.uk/government/statistics/adult-psychiatric-morbidity-survey-mental-health-and-wellbeing-england-2014 + source_data_url: |- + gAAAAABnRYjm-BZi3Nqs7In2gpZhuG03kWo5SiR_xJvXwncMzCmm--o4Z8FRFP55gIjTYun0AkKER23bHIUepBVraLszkW61lTi6JF3H7H0letw5MoLXJDe4S8UEwvkcq5h37pnxrcSJnzImSFrTQOuHF8l2kjJAF1QscQkYd_LGafvd9rWKkokdPPVLBf7iopl1gdm1gu7njbwKkauTdZyUbskGamH5fblFVs2LZv_GOGMQTv9yc4C-7dcXaRivHu-c-oU0j61D + date_accessed: '2024-11-26' + publication_year: 2016 + published_by: |- + "McManus S, Bebbington P, Jenkins R, Brugha T. (eds.) (2016) Mental health and wellbeing in England: Adult Psychiatric Morbidity Survey 2014. Leeds: NHS Digital" name: Current depression in England by age and gender (APMS, 2014) - description: This is a dataset of the prevalence of current depression in the general - population in England, living in private households. Households were sampled randomly - and individuals were interviewed using the revised Clinical Interview Schedule - (CIS-R), which is a diagnostic structured interview format to determine whether - people had common mental disorders in the past week. In this dataset, presence - of a current episode of major depression was determined. - source_name: Google Sheet - url: https://www.gov.uk/government/statistics/adult-psychiatric-morbidity-survey-mental-health-and-wellbeing-england-2014 - source_published_by: Google Sheet - source_data_url: https://docs.google.com/spreadsheets/d/e/2PACX-1vR8UNjna5w3mHijBNhbtk2vWPh_d2tYa4CrOZc2WFKeYP-ChcOCxNttPyv5izWcjnsZZv3CY2f98sd3/pub?output=csv - license_url: https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/ - license_name: Open Government Licence v3.0 + description: |- + This is a dataset of the prevalence of current depression in the general population in England, living in private households. Households were sampled randomly and individuals were interviewed using the revised Clinical Interview Schedule (CIS-R), which is a diagnostic structured interview format to determine whether people had common mental disorders in the past week. In this dataset, presence of a current episode of major depression was determined. + license: + name: Open Government Licence v3.0 + url: https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/ is_public: false - version: '2023-05-03' - publication_year: 2016 -wdir: ../../../data/snapshots/fasttrack/2023-05-03 outs: -- md5: 7a42c7cee959d16ae9d981f4b8b7ade1 - size: 288 - path: apms_2014.csv + - md5: 7a42c7cee959d16ae9d981f4b8b7ade1 + size: 288 + path: apms_2014.csv diff --git a/snapshots/fasttrack/2023-05-31/cholera.csv.dvc b/snapshots/fasttrack/2023-05-31/cholera.csv.dvc index f7eaa4c7aa4..c35236fed09 100644 --- a/snapshots/fasttrack/2023-05-31/cholera.csv.dvc +++ b/snapshots/fasttrack/2023-05-31/cholera.csv.dvc @@ -1,8 +1,11 @@ meta: - namespace: fasttrack - short_name: cholera - file_extension: csv - date_accessed: 2023-06-01 + source: + name: World Health Organization (2023) + url: nan + source_data_url: |- + https://docs.google.com/spreadsheets/d/e/2PACX-1vSlmIK_QFOjnb3DSPq6AoEeVFCPhsvKBE4yiUKrOogGhyu98SqoJutDR_3CEI43nvslvYIcJjICn-qI/pub?output=csv + date_accessed: '2024-11-26' + published_by: World Health Organization name: Cholera reported cases, deaths and case fatality rate (WHO, 2023) description: |- The data is created by combining multiple WHO Weekly Epidemiological Reports for cholera reported cases, deaths and case fatality rate. @@ -18,15 +21,8 @@ meta: 2020: https://web.archive.org/web/20230326231135/http://apps.who.int/iris/bitstream/handle/10665/345271/WER9637-445-454-eng-fre.pdf?sequence=1&isAllowed=y 2021: https://web.archive.org/web/20230526223955/https://apps.who.int/iris/bitstream/handle/10665/362858/WER9737-453-464-eng-fre.pdf?sequence=1&isAllowed=y - source_name: Google Sheet - url: '' - source_published_by: Google Sheet - source_data_url: - https://docs.google.com/spreadsheets/d/e/2PACX-1vSlmIK_QFOjnb3DSPq6AoEeVFCPhsvKBE4yiUKrOogGhyu98SqoJutDR_3CEI43nvslvYIcJjICn-qI/pub?output=csv - is_public: true - version: '2023-05-31' -wdir: ../../../data/snapshots/fasttrack/2023-05-31 + license: {} outs: -- md5: 035a7f6ada3274928cffda95091de105 - size: 4449 - path: cholera.csv + - md5: 39d2f683da32e151413ab8d2dbfc7582 + size: 4141 + path: cholera.csv diff --git a/snapshots/fasttrack/2023-06-16/guinea_worm.csv.dvc b/snapshots/fasttrack/2023-06-16/guinea_worm.csv.dvc index 37627bc762e..cb7263f0844 100644 --- a/snapshots/fasttrack/2023-06-16/guinea_worm.csv.dvc +++ b/snapshots/fasttrack/2023-06-16/guinea_worm.csv.dvc @@ -1,21 +1,16 @@ meta: - namespace: fasttrack - short_name: guinea_worm - file_extension: csv - date_accessed: 2023-06-16 + source: + name: The Carter Center (2023) + url: https://www.cartercenter.org/resources/pdfs/news/health_publications/guinea_worm/guinea-worm-cases-by-year-from-1989.pdf + source_data_url: |- + https://docs.google.com/spreadsheets/d/e/2PACX-1vQRR-nQzFpJXWz6cuB9aqrt2yt2Z8BtOtu3eNkF4WvI3xuNEdinb0vCN3wIagy2hEOHiSj3xUAffHeX/pub?output=csv + date_accessed: '2024-11-26' + publication_year: 2023 + published_by: The Carter Center (2023) name: Guinea Worm Cases - Carter Center (2023) description: The number of cases of guinea worm disease worldwide since 1989 - source_name: Google Sheet - url: - https://www.cartercenter.org/resources/pdfs/news/health_publications/guinea_worm/guinea-worm-cases-by-year-from-1989.pdf - source_published_by: Google Sheet - source_data_url: - https://docs.google.com/spreadsheets/d/e/2PACX-1vQRR-nQzFpJXWz6cuB9aqrt2yt2Z8BtOtu3eNkF4WvI3xuNEdinb0vCN3wIagy2hEOHiSj3xUAffHeX/pub?output=csv - is_public: true - version: '2023-06-16' - publication_year: 2023 -wdir: ../../../data/snapshots/fasttrack/2023-06-16 + license: {} outs: -- md5: 44af3d83549167d5edde31c323526c19 - size: 582 - path: guinea_worm.csv + - md5: 44af3d83549167d5edde31c323526c19 + size: 582 + path: guinea_worm.csv diff --git a/snapshots/fasttrack/2023-06-19/world_population_comparison.csv.dvc b/snapshots/fasttrack/2023-06-19/world_population_comparison.csv.dvc index 9e53e1e1f8a..d111694cd95 100644 --- a/snapshots/fasttrack/2023-06-19/world_population_comparison.csv.dvc +++ b/snapshots/fasttrack/2023-06-19/world_population_comparison.csv.dvc @@ -1,55 +1,31 @@ meta: + source: + name: Multiple sources compiled by Our World in Data (2019) + url: nan + source_data_url: |- + https://docs.google.com/spreadsheets/d/e/2PACX-1vQjyo0SCpkP7gW490fsxx2x0nkCPqW3elr5LfI-zbFWkb1rzOAumgJrEDO0eFpoEtPsZyHjIM58iXDe/pub?output=csv + date_accessed: '2024-12-02' + published_by: Multiple sources compiled by Our World in Data (2019) name: Historical world population comparison (various sources) - description: >- + description: |- Among others these are the original source: + McEvedy, Colin and Richard Jones, 1978, “Atlas of World Population History,” Facts on File, New York, pp. 342-351. - McEvedy, Colin and Richard Jones, 1978, “Atlas of World Population History,” Facts - on File, New York, pp. 342-351. - - - Biraben, Jean-Noel, 1980, An Essay Concerning Mankind’s Evolution, Population, - Selected Papers, December, table 2. - - - Durand, John D., 1974, “Historical Estimates of World Population: An Evaluation,” - University of Pennsylvania, Population - Center, Analytical and Technical Reports, Number 10, table 2. + Biraben, Jean-Noel, 1980, An Essay Concerning Mankind’s Evolution, Population, Selected Papers, December, table 2. + Durand, John D., 1974, “Historical Estimates of World Population: An Evaluation,” University of Pennsylvania, Population Center, Analytical and Technical Reports, Number 10, table 2. - Haub, Carl, 1995, “How Many People Have Ever Lived on Earth?” Population Today, - February, p. 5. + Haub, Carl, 1995, “How Many People Have Ever Lived on Earth?” Population Today, February, p. 5. + Thomlinson, Ralph, 1975, “Demographic Problems, Controversy Over Population Control,” Second Edition, Table 1. - Thomlinson, Ralph, 1975, “Demographic Problems, Controversy Over Population Control,” - Second Edition, Table 1. - - - United Nations, 1999, The World at Six Billion, Table 1, “World Population From” - Year 0 to Stabilization, p. 5, - + United Nations, 1999, The World at Six Billion, Table 1, “World Population From” Year 0 to Stabilization, p. 5, U.S. Census Bureau (USCB), 2012, Total Midyear Population for the World: 1950-2050. - - Michael Kremer (1993) “Population Growth and Technological Change: One Million - B.C. to 1990”, Quarterly Journal of Economics., - August 1993, pp.681-716. - source: - name: Google Sheet - description: - url: '' - source_data_url: - https://docs.google.com/spreadsheets/d/e/2PACX-1vQjyo0SCpkP7gW490fsxx2x0nkCPqW3elr5LfI-zbFWkb1rzOAumgJrEDO0eFpoEtPsZyHjIM58iXDe/pub?output=csv - owid_data_url: - date_accessed: '2023-07-03' - publication_date: - publication_year: - published_by: Google Sheet - license: - name: - url: -wdir: ../../../data/snapshots/fasttrack/2023-06-19 + Michael Kremer (1993) “Population Growth and Technological Change: One Million B.C. to 1990”, Quarterly Journal of Economics., August 1993, pp.681-716. + license: {} outs: -- md5: fa998c4590ac0bdada993b0bcaf0f2a8 - size: 14087 - path: world_population_comparison.csv + - md5: f7ff2acb39ad85d3a9b2599a63175abc + size: 13424 + path: world_population_comparison.csv diff --git a/snapshots/fasttrack/2023-08-07/pain_hours_days_hen_systems.csv.dvc b/snapshots/fasttrack/2023-08-07/pain_hours_days_hen_systems.csv.dvc index 6593646b1ed..4693ae861db 100644 --- a/snapshots/fasttrack/2023-08-07/pain_hours_days_hen_systems.csv.dvc +++ b/snapshots/fasttrack/2023-08-07/pain_hours_days_hen_systems.csv.dvc @@ -1,17 +1,16 @@ meta: source: - name: Google Sheet + name: Welfare Footprint based on Schuck-Paim and Alonso (2021) url: https://welfarefootprint.org/research-projects/laying-hens/ - source_data_url: + source_data_url: |- https://docs.google.com/spreadsheets/d/e/2PACX-1vRioMTHFwUUiCe2JWDOAZD7BDur4MSx3o576xfvTROzUY75sxIYqOZuIWQw_lopGkrLQ_lgXxMRgAk2/pub?output=csv - date_accessed: '2023-08-07' + date_accessed: '2024-11-26' publication_year: 2021 - published_by: Google Sheet + published_by: Welfare Footprint name: Pain hours and days of hen systems (Welfare Footprint) description: '' license: {} -wdir: ../../../data/snapshots/fasttrack/2023-08-07 outs: -- md5: 2053ef477157266c201a9c39c46a8df8 - size: 459 - path: pain_hours_days_hen_systems.csv + - md5: 2053ef477157266c201a9c39c46a8df8 + size: 459 + path: pain_hours_days_hen_systems.csv diff --git a/snapshots/fasttrack/2023-08-21/survey_livestock_oklahoma.csv.dvc b/snapshots/fasttrack/2023-08-21/survey_livestock_oklahoma.csv.dvc index 6a0b552ce7e..4e73c3ea3ea 100644 --- a/snapshots/fasttrack/2023-08-21/survey_livestock_oklahoma.csv.dvc +++ b/snapshots/fasttrack/2023-08-21/survey_livestock_oklahoma.csv.dvc @@ -1,18 +1,16 @@ meta: source: - name: Google Sheet - url: - https://web.archive.org/web/20190806000018/http://agecon.okstate.edu/files/january%202018.pdf - source_data_url: + name: Food Demand Survey, Oklahoma State University + url: https://web.archive.org/web/20190806000018/http://agecon.okstate.edu/files/january%202018.pdf + source_data_url: |- https://docs.google.com/spreadsheets/d/e/2PACX-1vTO7prhB6yU1jGOYi4ZvkBUV3wsfGi-ua3nXOE0fB5OpNH9YTClBw0H166ZbQ4VKhvv1ee3PhuUDeaB/pub?output=csv - date_accessed: '2023-08-22' - publication_year: '2018' - published_by: Google Sheet + date_accessed: '2024-11-26' + publication_year: 2018 + published_by: Oklahoma State University, Department of Agricultural Economics name: Survey attitudes to livestock farming (Oklahoma University) description: '' license: {} -wdir: ../../../data/snapshots/fasttrack/2023-08-21 outs: -- md5: b5d54c9c99616a0f61523365746d0e81 - size: 1195 - path: survey_livestock_oklahoma.csv + - md5: b5d54c9c99616a0f61523365746d0e81 + size: 1195 + path: survey_livestock_oklahoma.csv diff --git a/snapshots/fasttrack/2023-09-29/un_space_objects.csv.dvc b/snapshots/fasttrack/2023-09-29/un_space_objects.csv.dvc index babcf82a93d..c2033a5be3b 100644 --- a/snapshots/fasttrack/2023-09-29/un_space_objects.csv.dvc +++ b/snapshots/fasttrack/2023-09-29/un_space_objects.csv.dvc @@ -1,45 +1,25 @@ meta: source: - name: Google Sheet + name: United Nations Office for Outer Space Affairs, Online Index of Objects Launched into Outer Space (2023) url: https://www.unoosa.org/oosa/osoindex/search-ng.jspx - source_data_url: + source_data_url: |- https://docs.google.com/spreadsheets/d/e/2PACX-1vS0rsoIS37xrLx_JRHO2mdsqHmVreN255IRUSP1YLEP26NnIOz3bHLz5fDmR2bnTwwEY5gttb17fZgU/pub?output=csv - date_accessed: '2023-09-29' - publication_year: '2023' - published_by: Google Sheet + date_accessed: '2024-11-26' + publication_year: 2023 + published_by: Online Index of Objects Launched into Outer Space (2023) name: Online Index of Objects Launched into Outer Space (UN, 2023-09-29) - description: >- - This data is compiled from the Online Index of Objects Launched into Outer Space, - maintained by the United Nations Office - for Outer Space Affairs. + description: |- + This data is compiled from the Online Index of Objects Launched into Outer Space, maintained by the United Nations Office for Outer Space Affairs. + Since 1962, the United Nations has maintained a Register of Objects Launched into Outer Space. Originally established as a mechanism to aid the United Nations Committee on the Peaceful Uses of Outer Space in its discussions on the political, legal, and technical issues concerning outer space, the evolution of international space law resulted in space object registration becoming a means of identifying which States' bear international responsibility and liability for space objects. - Since 1962, the United Nations has maintained a Register of Objects Launched into - Outer Space. Originally established - as a mechanism to aid the United Nations Committee on the Peaceful Uses of Outer - Space in its discussions on the political, - legal, and technical issues concerning outer space, the evolution of international - space law resulted in space object - registration becoming a means of identifying which States' bear international - responsibility and liability for space objects. - - - The source indicates that around 87% of all satellites, probes, landers, crewed - spacecraft, and space station flight elements - launched into Earth orbit or beyond have been registered with the Secretary-General. - + The source indicates that around 87% of all satellites, probes, landers, crewed spacecraft, and space station flight elements launched into Earth orbit or beyond have been registered with the Secretary-General. In the data shown on our charts: - - - when an object is launched by a country on behalf of another one, it is attributed - to the latter; - - - when a launch is made jointly by several countries, it is recorded in each of - these countries' time series, but only - once in the 'World' series. + - when an object is launched by a country on behalf of another one, it is attributed to the latter; + - when a launch is made jointly by several countries, it is recorded in each of these countries' time series, but only once in the 'World' series. license: {} -wdir: ../../../data/snapshots/fasttrack/2023-09-29 outs: -- md5: 44879ab7f57a44915bcd36b4b7833fd5 - size: 22610 - path: un_space_objects.csv + - md5: 44879ab7f57a44915bcd36b4b7833fd5 + size: 22610 + path: un_space_objects.csv diff --git a/snapshots/fasttrack/2023-10-05/great_pacific_garbage_lebreton.csv.dvc b/snapshots/fasttrack/2023-10-05/great_pacific_garbage_lebreton.csv.dvc index 5030249d9f8..96ec46a4fe7 100644 --- a/snapshots/fasttrack/2023-10-05/great_pacific_garbage_lebreton.csv.dvc +++ b/snapshots/fasttrack/2023-10-05/great_pacific_garbage_lebreton.csv.dvc @@ -1,17 +1,17 @@ meta: source: - name: Google Sheet + name: Plastics in Great Pacific Garbage Patch (Lebreton et al. 2022) url: https://www.nature.com/articles/s41598-022-16529-0 - source_data_url: + source_data_url: |- https://docs.google.com/spreadsheets/d/e/2PACX-1vShK8-lh5FUIl954ziCnKzkt9N3B5IdxAxk9mh-QhufRD-SuovXXLKjbQtq8g40yzUbHIPWRVLuVzKZ/pub?output=csv - date_accessed: '2023-10-05' - publication_year: '2022' - published_by: Google Sheet + date_accessed: '2024-11-26' + publication_year: 2022 + published_by: |- + Lebreton et al. (2022). Industrialised fishing nations largely contribute to floating plastic pollution in the North Pacific subtropical gyre. Nature Scientific Reports. name: Plastics in Great Pacific Garbage Patch (Lebreton) description: '' license: {} -wdir: ../../../data/snapshots/fasttrack/2023-10-05 outs: -- md5: 96e4d3017bd50b72a8bd267198ab1971 - size: 338 - path: great_pacific_garbage_lebreton.csv + - md5: 96e4d3017bd50b72a8bd267198ab1971 + size: 338 + path: great_pacific_garbage_lebreton.csv diff --git a/snapshots/fasttrack/latest/antimicrobial_usage_livestock.csv.dvc b/snapshots/fasttrack/latest/antimicrobial_usage_livestock.csv.dvc index 2ee12981302..e904f870cd9 100644 --- a/snapshots/fasttrack/latest/antimicrobial_usage_livestock.csv.dvc +++ b/snapshots/fasttrack/latest/antimicrobial_usage_livestock.csv.dvc @@ -1,33 +1,23 @@ meta: + origin: + producer: Mulchandani et al. + title: 'Global trends in antimicrobial use in food-producing animals: 2020 to 2030' + description: |- + Data on usage of antimicrobials in food animals were collected from 42 countries. Multivariate regression models were used in combination with projections of animal counts for cattle, sheep, chicken, and pigs from the Food and Agriculture Organization to estimate global antimicrobial usage of veterinary antimicrobials in 2020 and 2030. Maps of animal densities were used to identify geographic hotspots of antimicrobial use. In each country, estimates of antimicrobial use (tonnes) were calibrated to match continental-level reports of antimicrobial use intensity (milligrams per kilogram of animal) from the World Organization for Animal Health, as well as country-level reports of antimicrobial use from countries that made this information publicly available. + citation_full: |- + Mulchandani, R., Wang, Y., Gilbert, M., & Van Boeckel, T. P. (2023). Global trends in antimicrobial use in food-producing animals: 2020 to 2030. PLOS Global Public Health, 3(2), e0001305. https://doi.org/10.1371/journal.pgph.0001305 + url_main: https://journals.plos.org/globalpublichealth/article?id=10.1371/journal.pgph.0001305 + url_download: |- + https://docs.google.com/spreadsheets/d/e/2PACX-1vT1GgT43B-J5fD0kqup2QeajeMNLtjo10An4N3OkugtbOn-Q4OIaoI5pC2hsnYroRn8UmVhcczZADHw/pub?output=csv + date_accessed: '2024-12-06' + date_published: '2023-02-01' + license: + name: Open access name: Antimicrobial usage in livestock - description: This dataset estimates the usage of antimicrobials in livestock (cattle, - sheep, chicken, and pigs) by country. Data on antimicrobials comes from government - reports, surveillance systems and national surveys. In addition, the authors estimate - the biomass of livestock in the country, to adjust for differences in antimicrobial - usage by animal size. Biomass data comes from the Food and Agriculture Organization - (FAO). 'The PCU represents the total number of animals in a country (alive or - slaughtered), multiplied by the average weight of the animal at the time of treatment. - Therefore, the PCU is a standardization metric that accounts for differences in - animal weight, and number of production cycles per year between countries.' Therefore, - mg/PCU refers to the usage of antimicrobials per animal population-corrected unit. - source: - name: Google Sheet - description: - url: - https://journals.plos.org/globalpublichealth/article?id=10.1371/journal.pgph.0001305 - source_data_url: - gAAAAABkx3_ZrECpU1RI6tcJPiqDvqB2RKAZEsofB7GToYg6Vsw-LANlBdEgn1IxkrXt-La91vsraylCA313XJ_3HlggZ9aLs42krkNJwtDpi8I31JdQoiZg5lPB_m9w4Kx0dab4AgKjPhsnX8Y1-YoOw0-uSgkyigp2-gdc0mBZ6XknkcRp-53G1AZamnOn-9p8kKzkuzmqYv6ISc3tPI12oPqWZSBSM1UzFW1QRmM8VwTqtE1WqAGly3iX3yIPQUzV_E5tu2vI - owid_data_url: - date_accessed: '2023-07-31' - publication_date: - publication_year: 2023 - published_by: Google Sheet - license: - name: - url: - is_public: false -wdir: ../../../data/snapshots/fasttrack/latest + description: |- + This dataset estimates the usage of antimicrobials in livestock (cattle, sheep, chicken, and pigs) by country. Data on antimicrobials comes from government reports, surveillance systems and national surveys. In addition, the authors estimate the biomass of livestock in the country, to adjust for differences in antimicrobial usage by animal size. Biomass data comes from the Food and Agriculture Organization (FAO). 'The PCU represents the total number of animals in a country (alive or slaughtered), multiplied by the average weight of the animal at the time of treatment. Therefore, the PCU is a standardization metric that accounts for differences in animal weight, and number of production cycles per year between countries.' Therefore, mg/PCU refers to the usage of antimicrobials per animal population-corrected unit. + license: {} outs: -- md5: 2e79d5ae8ead88349c544d2e7b4b5ca2 - size: 9186 - path: antimicrobial_usage_livestock.csv + - md5: e0c44fec35851446ebb61784ce6528e3 + size: 8682 + path: antimicrobial_usage_livestock.csv diff --git a/snapshots/fasttrack/latest/cumulative_lives_saved_vaccination_shattock.csv.dvc b/snapshots/fasttrack/latest/cumulative_lives_saved_vaccination_shattock.csv.dvc index 8c9c6de7258..6fee0afab30 100644 --- a/snapshots/fasttrack/latest/cumulative_lives_saved_vaccination_shattock.csv.dvc +++ b/snapshots/fasttrack/latest/cumulative_lives_saved_vaccination_shattock.csv.dvc @@ -1,14 +1,14 @@ meta: origin: producer: |- - Shattock et al. (2024). Contribution of vaccination to improved child survival: modelling 50 years of the Expanded Programme on Immunization. + Shattock et al. (2024). Contribution of vaccination to improved survival and health: modelling 50 years of the Expanded Programme on Immunization. title: cumulative_lives_saved_vaccination_shattock citation_full: |- - Shattock et al. (2024). Contribution of vaccination to improved child survival: modelling 50 years of the Expanded Programme on Immunization. The Lancet. + Shattock et al. (2024). Contribution of vaccination to improved survival and health: modelling 50 years of the Expanded Programme on Immunization. The Lancet. url_main: https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(24)00850-X/fulltext url_download: |- https://docs.google.com/spreadsheets/d/e/2PACX-1vQHxzyufogWkSCuwEmHd6jF2c3JfqTdUY9ngwgpdwZfR5FA7JGQRzKNKhJ6hmAfGXReeAmgpHRMj8iM/pub?output=csv - date_accessed: '2024-05-03' + date_accessed: '2024-11-28' name: Cumulative lives saved from vaccinations since 1974 (Shattock et al. 2024) description: The cumulative number of lives saved thanks to vaccinations, from 1974 onwards. license: {} diff --git a/snapshots/fasttrack/latest/democracy_freedom_house.csv.dvc b/snapshots/fasttrack/latest/democracy_freedom_house.csv.dvc index 54fb121dd16..578a16d565f 100644 --- a/snapshots/fasttrack/latest/democracy_freedom_house.csv.dvc +++ b/snapshots/fasttrack/latest/democracy_freedom_house.csv.dvc @@ -1,25 +1,20 @@ meta: source: - name: Google Sheet + name: Freedom House (2023) url: https://freedomhouse.org/report/freedom-world - source_data_url: + source_data_url: |- https://docs.google.com/spreadsheets/d/e/2PACX-1vRXOAxs_t6f-Y3kMGmKc0trQiH2PMWjOxxlHZ1uKd88g_sFMq1zvgaeGO3G15lJdSixh9dCIIqUcBHI/pub?output=csv - date_accessed: '2023-09-05' - published_by: Google Sheet + date_accessed: '2024-11-26' + published_by: Freedom House (2023). Freedom in the World. name: Democracy - Freedom House (2023) - description: >- - This dataset provides information on political regimes, using data from Freedom - House's Freedom in the World (2023). - + description: |- + This dataset provides information on political regimes, using data from Freedom House's Freedom in the World (2023). You can read a description of the data in this post: https://ourworldindata.org/democracies-measurement - - You can download the code and complete dataset, including supplementary variables, - from GitHub: https://github.com/owid/notebooks/tree/main/BastianHerre/democracy + You can download the code and complete dataset, including supplementary variables, from GitHub: https://github.com/owid/notebooks/tree/main/BastianHerre/democracy license: {} -wdir: ../../../data/snapshots/fasttrack/latest outs: -- md5: 5407777c1c32eae16bdc68a7dab2dcef - size: 629063 - path: democracy_freedom_house.csv + - md5: 8c5d63ec87bb5475b75de261d7e32a73 + size: 629913 + path: democracy_freedom_house.csv diff --git a/snapshots/fasttrack/latest/gbd_2019_mental_health_country_coverage.csv.dvc b/snapshots/fasttrack/latest/gbd_2019_mental_health_country_coverage.csv.dvc index eaf8778537e..723cdb77c01 100644 --- a/snapshots/fasttrack/latest/gbd_2019_mental_health_country_coverage.csv.dvc +++ b/snapshots/fasttrack/latest/gbd_2019_mental_health_country_coverage.csv.dvc @@ -1,26 +1,22 @@ meta: - namespace: fasttrack - short_name: gbd_2019_mental_health_country_coverage - file_extension: csv - date_accessed: 2023-05-29 + source: + name: IHME GBD (2019) + url: https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(20)30925-9/fulltext + source_data_url: |- + gAAAAABnRYlazzL_oEfuzBXSmjRVB2Y_aOmXSCAXVYIW_72BvzIYsW9GUt8OjdfLueLZUKFP-ZGq7e8B19xEzO3nPErc9S9x8w5MaFQeUEEwgZHOHjHGCjfOxrcUDXEQ-RyBwZuKmo7Ngy5dTDkh_sza4x9FOhmx5yUB25NI0TKhPhkPxrC_AIads-3sWMviypayPf_OebJnlMZUgRBEnRYqUH5sGcQTZvpi2ixly61NvwF_oU1_8FZL8iyj9mDiUvBP-kpERU3L + date_accessed: '2024-11-26' + publication_year: 2020 + published_by: |- + Vos, T., Lim, S. S., Abbafati, C., Abbas, K. M., Abbasi, M., Abbasifard, M., Abbasi-Kangevari, M., Abbastabar, H., Abd-Allah, F., Abdelalim, A., Abdollahi, M., Abdollahpour, I., Abolhassani, H., Aboyans, V., Abrams, E. M., Abreu, L. G., Abrigo, M. R. M., Abu-Raddad, L. J., Abushouk, A. I., … Murray, C. J. L. (2020). Global burden of 369 diseases and injuries in 204 countries and territories, 1990–2019: A systematic analysis for the Global Burden of Disease Study 2019. The Lancet, 396(10258), 1204–1222. name: Countries with mental health data in GBD 2019 description: |- Dataset showing the number of countries with primary data on the prevalence of mental illnesses. These were found after a systematic review, grey literature search and expert consultation, to identify studies with data on the prevalence of each mental illness. 'The GBD inclusion criteria stipulated that: (1) the diagnostic criteria must be from 1980 onward; (2) “caseness” must be based on clinical threshold as established by the DSM, ICD, Chinese Classification of Mental Disorders (CCMD), or diagnosed by a clinician using established tools; (3) sufficient information must be provided on study method and sample characteristics to assess the quality of the study; and (4) study samples must be representative of the general population (i.e., case studies, veterans, or refugee samples were excluded). No limitation was set on the language of publication.' - source_name: Google Sheet - url: - https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(20)30925-9/fulltext - source_published_by: Google Sheet - source_data_url: - gAAAAABkdGlgpHgXXTDAwptybyG87JbjFVjs02Q-Rzc9mvCEIbiBXzn73TAFhhJhTIRWevZe9DtC6nMtJ1Cq1aUvY-ZA7cYHUb3afdxTpm9C-vyTkjYLqSy7VlBPzYJODjHtY-78465q_2SAHLZ1LJC2UQWtGtzHzWszyyTlsspAbhEpata6QVf2_DDdwJpHdWnJBVdGqnddeqpSKtwYXmoIG-A8BX4FAp5eVmbAzuhwR2b5vavEHmTnIJPyMOVBCu4vp9sGudYb - license_url: - https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(20)30925-9/fulltext + license: + url: https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(20)30925-9/fulltext is_public: false - version: latest - publication_year: 2020 -wdir: ../../../data/snapshots/fasttrack/latest outs: -- md5: f972f0ff8f16acaf3af8c63d4e880e96 - size: 486 - path: gbd_2019_mental_health_country_coverage.csv + - md5: f972f0ff8f16acaf3af8c63d4e880e96 + size: 486 + path: gbd_2019_mental_health_country_coverage.csv diff --git a/snapshots/fasttrack/latest/global_maternal_offspring_loss.csv.dvc b/snapshots/fasttrack/latest/global_maternal_offspring_loss.csv.dvc index ef84d64a187..0aa1753a193 100644 --- a/snapshots/fasttrack/latest/global_maternal_offspring_loss.csv.dvc +++ b/snapshots/fasttrack/latest/global_maternal_offspring_loss.csv.dvc @@ -1,24 +1,18 @@ meta: source: - name: Google Sheet + name: Smith-Greenaway et al. (2021) url: https://gh.bmj.com/content/6/4/e004837.abstract - source_data_url: + source_data_url: |- https://docs.google.com/spreadsheets/d/e/2PACX-1vSiCgGTJ8kXoVPpaaskyRz79os2jbWtLbcqk18ybEXI_tsE6WPJy6DH-lXggK_VbHir456mI3D98Jbd/pub?output=csv - date_accessed: '2023-09-05' - publication_year: '2021' - published_by: Google Sheet + date_accessed: '2024-11-26' + publication_year: 2021 + published_by: |- + Global burden of maternal bereavement: indicators of the cumulative prevalence of child loss. (2021) Emily Smith-Greenaway, Diego Alburez-Gutierrez, Jenny Trinitapoli, Emilio Zagheni. name: Global maternal offspring loss - Smith-Greenaway et al. 2021 - description: This dataset shows survey data and estimates of maternal offspring - loss across countries. This includes mothers who have lost an infant, child under - 5 years old, or offspring. These are given as a rate per 1000 women in the age - group. Underlying data comes from large-scale surveys (such as the Demographic - and Health Surveys and Multiple Indicator Cluster Surveys) conducted in many low- - and middle-income countries. For countries lacking data, these are estimated using - an indirect approach that combines formal kinship models and life-table methods - in an additional 81 countries. + description: |- + This dataset shows survey data and estimates of maternal offspring loss across countries. This includes mothers who have lost an infant, child under 5 years old, or offspring. These are given as a rate per 1000 women in the age group. Underlying data comes from large-scale surveys (such as the Demographic and Health Surveys and Multiple Indicator Cluster Surveys) conducted in many low- and middle-income countries. For countries lacking data, these are estimated using an indirect approach that combines formal kinship models and life-table methods in an additional 81 countries. Citation: Smith-Greenaway, E., Alburez-Gutierrez, D., Trinitapoli, J., & Zagheni, E. (2021). Global burden of maternal bereavement: Indicators of the cumulative prevalence of child loss. BMJ Global Health, 6(4), e004837. https://doi.org/10.1136/bmjgh-2020-004837 license: {} -wdir: ../../../data/snapshots/fasttrack/latest outs: -- md5: 7ce7e9d8a371afa038a470f8977d160d - size: 8075 - path: global_maternal_offspring_loss.csv + - md5: 7ce7e9d8a371afa038a470f8977d160d + size: 8075 + path: global_maternal_offspring_loss.csv diff --git a/snapshots/fasttrack/latest/historical_france_mortality_cause.csv.dvc b/snapshots/fasttrack/latest/historical_france_mortality_cause.csv.dvc index 9305707e4cf..89c8a92d726 100644 --- a/snapshots/fasttrack/latest/historical_france_mortality_cause.csv.dvc +++ b/snapshots/fasttrack/latest/historical_france_mortality_cause.csv.dvc @@ -1,65 +1,36 @@ meta: source: - name: Google Sheet + name: Institut National d'Études Démographiques url: https://www.demographic-research.org/Volumes/Vol36/21/ - source_data_url: - https://docs.google.com/spreadsheets/d/e/2PACX-1vSUInsCt97X8kjDGENSZm5-xLSbE_P4MEAW4bYkGHn-KN5aVWQ4VAm3JzziskCUTkFQuWQVinWtLO_I/pub?output=csv - date_accessed: '2023-08-18' - publication_year: '2014' - published_by: Google Sheet + source_data_url: |- + gAAAAABnRYkUiIe1Lp-eC1kj1PYYYQ25kHCvpjl7pYYYL4eO5R8EtQ_r17u121vjvLUCpJ3X0LDrvoZ7IGgLK52Of2eKqxKBxWh30Ud4jSt_oSgpu6iL2FdM4M5nJbVStK7IBlb2SregUwmWCdJDn8xUpzz__p-Ly3mpfWJfa1i11msoZyXQVbTlcGbq5Rz9GEYok3B9elGj9tKPOFgqprbt4_UgHuYSUGPyn9fXCX9DBkRhC52YfXIpd_G625e2UxdS1-TK06j4 + date_accessed: '2024-11-26' + publication_year: 2014 + published_by: Jacques Vallin and France Meslé name: Database on causes of death in France from 1925 to 1999 - description: >- - Dataset on mortality rates from each cause of death category in France between - 1925 and 1999. The underlying data for - this chart comes from the Institut National d'Études Démographiques, published - by Jacques Vallin and France Meslé, and - covers causes of deaths nationally in France between 1925 and 1999. Causes of - death were categorized into categories according - to the 9th edition of the International Classification of Diseases (ICD-9) manual. - Mortality rates are given for five-year - age bands, as an annual rate out of 100,000 people in that age group. Below are - the ICD codes used for each cause category: - All causes = 000*-999*, - + description: |- + Dataset on mortality rates from each cause of death category in France between 1925 and 1999. The underlying data for this chart comes from the Institut National d'Études Démographiques, published by Jacques Vallin and France Meslé, and covers causes of deaths nationally in France between 1925 and 1999. Causes of death were categorized into categories according to the 9th edition of the International Classification of Diseases (ICD-9) manual. Mortality rates are given for five-year age bands, as an annual rate out of 100,000 people in that age group. Below are the ICD codes used for each cause category: All causes = 000*-999*, Infectious and parasitic diseases = 001*-139*, - Neoplasms = 140*-239*, - Endocrine nutritional and metabolic diseases and immunity disorders = 240*-279*, - Diseases of the blood and blood-forming organs = 280*-289*, - Mental disorders = 290*-319*, - Diseases of the nervous system = 320*-359*, - Diseases of the sense organs = 360*-389*, - Diseases of the circulatory system = 390*-459*, - Diseases of the respiratory system = 460*-519*, - Diseases of the digestive system = 520*-579*, - Diseases of the genitourinary system = 580*-629*, - Complications of pregnancy childbirth and the puerperium = 630*-679*, - Diseases of the skin and subcutaneous tissue = 680*-709*, - Diseases of the musculoskeletal system and connective tissue = 710*-739*, - Congenital anomalies = 740*-759*, - Certain conditions originating in the perinatal period = 760*-779*, - Symptoms signs and ill-defined conditions = 780*-799*, - External causes (injury and poisoning) = 800*-999* license: {} is_public: false -wdir: ../../../data/snapshots/fasttrack/latest outs: -- md5: 856b79cf2b8b9e0ba868c7d4084c7c4a - size: 629827 - path: historical_france_mortality_cause.csv + - md5: 856b79cf2b8b9e0ba868c7d4084c7c4a + size: 629827 + path: historical_france_mortality_cause.csv diff --git a/snapshots/fasttrack/latest/infant_mortality_vaccination_shattock.csv.dvc b/snapshots/fasttrack/latest/infant_mortality_vaccination_shattock.csv.dvc index 9f6c72e61cb..c93ee0e7a2c 100644 --- a/snapshots/fasttrack/latest/infant_mortality_vaccination_shattock.csv.dvc +++ b/snapshots/fasttrack/latest/infant_mortality_vaccination_shattock.csv.dvc @@ -1,14 +1,14 @@ meta: origin: producer: |- - Shattock et al. (2024). Contribution of vaccination to improved child survival: modelling 50 years of the Expanded Programme on Immunization. + Shattock et al. (2024). Contribution of vaccination to improved survival and health: modelling 50 years of the Expanded Programme on Immunization. title: infant_mortality_vaccination_shattock citation_full: |- - Shattock et al. (2024). Contribution of vaccination to improved child survival: modelling 50 years of the Expanded Programme on Immunization. The Lancet. + Shattock et al. (2024). Contribution of vaccination to improved survival and health: modelling 50 years of the Expanded Programme on Immunization. The Lancet. url_main: https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(24)00850-X/fulltext url_download: |- https://docs.google.com/spreadsheets/d/e/2PACX-1vSLcIzJwPFHDM0x41c0X0h4_YNiw43W3YtVQfpoNtVKOjCZuOyyFcd0JM1O-RjCW_DASDOJkWRIfntY/pub?output=csv - date_accessed: '2024-05-03' + date_accessed: '2024-11-28' name: Infant mortality with and without vaccinations (Shattock et al, 2024) description: Estimates of the infant mortality rates with and without vital vaccinations. license: {} diff --git a/snapshots/fasttrack/latest/lead_paint_regulation_who.csv.dvc b/snapshots/fasttrack/latest/lead_paint_regulation_who.csv.dvc index b07698a5666..a45b6985906 100644 --- a/snapshots/fasttrack/latest/lead_paint_regulation_who.csv.dvc +++ b/snapshots/fasttrack/latest/lead_paint_regulation_who.csv.dvc @@ -1,24 +1,20 @@ meta: - namespace: fasttrack - short_name: lead_paint_regulation_who - file_extension: csv - date_accessed: 2023-05-31 + source: + name: Lead paint regulations (WHO, 2023) + url: https://www.who.int/data/gho/data/themes/topics/indicator-groups/legally-binding-controls-on-lead-paint + source_data_url: |- + gAAAAABnRY1UxjQRqqtwi3k4YRi5i2a_AmviR0K1Yq7lWpp5mQ8zmhp0sOt2x_D-aN8W2WkMf8tCvbcZ2Bi41ergSpO1KCkDtsaTcT24IcGVC16eX9TpV0AkYs4TAXg4LbY31u0XeecK9mQ7uw7nzqkQLsEbEDXTz7izPL1kz58WSwNQE8WCvzhV2kv3mddI_ycEwFGUFSypG0QXyYHreQLjP3n5CsnC2vGymzKGwdQUmiK0r2nipPcCj4vh3O6qbvSj0W_hcA5j + date_accessed: '2024-11-26' + publication_year: 2023 + published_by: World Health Organization (WHO) name: Lead paint regulations (WHO, 2023) description: |- The WHO collects data on which countries have legally-binding controls on lead paint. It sources this data from surveys conducted by WHO and UNEP of national authorities. The World Health Organization (WHO) tracks the introduction of legally-binding controls on lead concentrations in paint. Paint is a main contributor to harmful lead exposure. The stringency of controls on lead paint can vary by country. Maximum concentrations of lead can differ, and may only apply to particular types of paint (for example, products used in households). - source_name: Google Sheet - url: - https://www.who.int/data/gho/data/themes/topics/indicator-groups/legally-binding-controls-on-lead-paint - source_published_by: Google Sheet - source_data_url: - gAAAAABkdy33dfUqbIoFH4JceoCGzD0jhxgG7WWfCuXNbU3fqjWI0eFoNybQuugGLBO2bR-HGEAAMlzhGcAweillVjqC8fv2ZuiG5kSeLgbLGBbZc2AiBucQz5QwuPOh-_mMD0qMPTnsiDvHsfGp91drLXO4da3usmBrcIJbn0vv2bknY0iuDYbnyOf4HKHcbLM_9iazUXMeIvIzuvEHq3iu4QZHaTHgbP66JFYwGhM0EiP_mxGKNbart9M1j3zVkXRESjdxKTnA + license: {} is_public: false - version: latest - publication_year: 2023 -wdir: ../../../data/snapshots/fasttrack/latest outs: -- md5: 1699f03673a329af3bf555a0b4e6e8a0 - size: 2935 - path: lead_paint_regulation_who.csv + - md5: 1699f03673a329af3bf555a0b4e6e8a0 + size: 2935 + path: lead_paint_regulation_who.csv diff --git a/snapshots/fasttrack/latest/lives_saved_vaccination_who.csv.dvc b/snapshots/fasttrack/latest/lives_saved_vaccination_who.csv.dvc index aa7be1be0f4..0a344e05f02 100644 --- a/snapshots/fasttrack/latest/lives_saved_vaccination_who.csv.dvc +++ b/snapshots/fasttrack/latest/lives_saved_vaccination_who.csv.dvc @@ -1,14 +1,14 @@ meta: origin: producer: |- - Shattock et al. (2024). Contribution of vaccination to improved child survival: modelling 50 years of the Expanded Programme on Immunization. + Shattock et al. (2024). Contribution of vaccination to improved survival and health: modelling 50 years of the Expanded Programme on Immunization. title: lives_saved_vaccination_who citation_full: |- - Shattock et al. (2024). Contribution of vaccination to improved child survival: modelling 50 years of the Expanded Programme on Immunization. The Lancet. + Shattock et al. (2024). Contribution of vaccination to improved survival and health: modelling 50 years of the Expanded Programme on Immunization. The Lancet. url_main: https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(24)00850-X/fulltext url_download: |- https://docs.google.com/spreadsheets/d/e/2PACX-1vSP20L-mjOB7xZ73EGhXyjZSdC_CBwWZAKkCs_T4iCRgwXKNXBb0K2H8L8mI7rnLIol8MqYAyG5YMHq/pub?output=csv - date_accessed: '2024-05-03' + date_accessed: '2024-11-28' name: Lives saved by vaccinations (WHO, 2024) description: Estimates of the number of lives saved by vaccination over the last 50 years (from 1974 to 2024). license: {} diff --git a/snapshots/fasttrack/latest/nuclear_warhead_inventories.csv.dvc b/snapshots/fasttrack/latest/nuclear_warhead_inventories.csv.dvc index c5d684cbde4..74c09663476 100644 --- a/snapshots/fasttrack/latest/nuclear_warhead_inventories.csv.dvc +++ b/snapshots/fasttrack/latest/nuclear_warhead_inventories.csv.dvc @@ -1,26 +1,19 @@ meta: source: - name: Google Sheet + name: Federation of American Scientists (2023) url: https://fas.org/issues/nuclear-weapons/status-world-nuclear-forces/ - source_data_url: + source_data_url: |- https://docs.google.com/spreadsheets/d/e/2PACX-1vRLseg03w2pyyqQIq9ip016J168lYsJ8BWJRqzDX8L8LqF6zm4TTyOGS7HmrwW3ZL1bj2KHXQNuhNir/pub?output=csv - date_accessed: '2023-08-22' - published_by: Google Sheet + date_accessed: '2024-11-26' + published_by: Hans M. Kristensen, Matt Korda, Eliana Reynolds, and Robert Norris. name: Nuclear warhead inventories – Federation of American Scientists - description: >- - This dataset provides information on the nuclear warhead inventories by the nuclear - powers, using data from the Federation - of American Scientists, prepared by Hans M. Kristensen, Matt Korda, and Robert - Norris. - - - You can download the code and complete dataset, including supplementary variables, - from GitHub: https://github.com/owid/notebooks/tree/main/BastianHerre/nuclear_weapons + description: |- + This dataset provides information on the nuclear warhead inventories by the nuclear powers, using data from the Federation of American Scientists, prepared by Hans M. Kristensen, Matt Korda, and Robert Norris. + You can download the code and complete dataset, including supplementary variables, from GitHub: https://github.com/owid/notebooks/tree/main/BastianHerre/nuclear_weapons Publisher source: None license: {} -wdir: ../../../data/snapshots/fasttrack/latest outs: -- md5: e1269c0f219efe73d1c408caa13c3ba8 - size: 694 - path: nuclear_warhead_inventories.csv + - md5: e1269c0f219efe73d1c408caa13c3ba8 + size: 694 + path: nuclear_warhead_inventories.csv diff --git a/snapshots/fasttrack/latest/nuclear_warhead_stockpiles.csv.dvc b/snapshots/fasttrack/latest/nuclear_warhead_stockpiles.csv.dvc index fbb3fe5c9cb..12f4d0b0114 100644 --- a/snapshots/fasttrack/latest/nuclear_warhead_stockpiles.csv.dvc +++ b/snapshots/fasttrack/latest/nuclear_warhead_stockpiles.csv.dvc @@ -1,26 +1,19 @@ meta: source: - name: Google Sheet + name: Federation of American Scientists (2023) url: https://fas.org/issues/nuclear-weapons/status-world-nuclear-forces/ - source_data_url: + source_data_url: |- https://docs.google.com/spreadsheets/d/e/2PACX-1vQ9gi-5QBEFuGclzGkfwv9NY_DavklY4DmA9-6QFG7KVFXTvuDf6fe4ETT7_e2zko4akSlAAqGduasG/pub?output=csv - date_accessed: '2023-10-16' - published_by: Google Sheet + date_accessed: '2024-11-26' + published_by: Hans M. Kristensen, Matt Korda, Eliana Reynolds, and Robert Norris. name: Nuclear warhead stockpiles – Federation of American Scientists - description: >- - This dataset provides information on the number of stockpiled nuclear warheads - by the nuclear powers, using data from - the Federation of American Scientists, prepared by Hans M. Kristensen, Matt Korda, - Eliana Reynolds, and Robert Norris. - - - You can download the code and complete dataset, including supplementary variables, - from GitHub: https://github.com/owid/notebooks/tree/main/BastianHerre/nuclear_weapons + description: |- + This dataset provides information on the number of stockpiled nuclear warheads by the nuclear powers, using data from the Federation of American Scientists, prepared by Hans M. Kristensen, Matt Korda, Eliana Reynolds, and Robert Norris. + You can download the code and complete dataset, including supplementary variables, from GitHub: https://github.com/owid/notebooks/tree/main/BastianHerre/nuclear_weapons Publisher source: None license: {} -wdir: ../../../data/snapshots/fasttrack/latest outs: -- md5: 3ae0bc86faefa1734117ba84f4761adc - size: 15471 - path: nuclear_warhead_stockpiles.csv + - md5: 3ae0bc86faefa1734117ba84f4761adc + size: 15471 + path: nuclear_warhead_stockpiles.csv diff --git a/snapshots/fasttrack/latest/pain_hours_hen_systems.csv.dvc b/snapshots/fasttrack/latest/pain_hours_hen_systems.csv.dvc index 78ffe32d43b..b9989419fb6 100644 --- a/snapshots/fasttrack/latest/pain_hours_hen_systems.csv.dvc +++ b/snapshots/fasttrack/latest/pain_hours_hen_systems.csv.dvc @@ -1,23 +1,17 @@ meta: - name: Pain hours of hen systems (Welfare Footprint) - description: '' source: - name: Google Sheet - description: + name: Welfare Footprint based on Schuck-Paim and Alonso (2021) url: https://welfarefootprint.org/research-projects/laying-hens/ - source_data_url: - gAAAAABkwT3GHig9-hHZr_XHHeWuAsK5PHjtz55H-Tvu_KCW-XytOuBUvNMnrv3hZ593VSyU95XV8hLYTSto8khid8slVfs4OA7joXQ4qzAsufp7A3vltFcf9o3BbCJseT64F2CYUUgMkJ9ptK0YjR0E2VhxNWylRc5G8OCv19srGHlFBGjG7_isIrTbJimw3L_c4hmRMlUNEZMMoo9OyTNjNRKb6Ta_4bkYL4xn6hBHt-89D_6xF58zl2ZQFdtjZn69vnwBW58p - owid_data_url: - date_accessed: '2023-07-26' - publication_date: + source_data_url: |- + gAAAAABnRY4A9xP89q46o72YIhvw5MBBfzt_84o2BNljMcBv29JHUFXvVtuYbTDPVsVbX-hfm9rZ8DJ-rW8qWqW0lWbhRDNvNd_ncCTITnovArjMEeLU41PNBF1vXiAz3Exdbb4orqamK0KosiZFwIKGtyvRFrF7XOH32d6O3z8ybbYchAUPJ8fqDyHP87olHyJt21OraPoNDQZ-MmNI5oSAHeMlz6zqyaEtO6mfd1g98R0B5QG6wf4zx7ogpunekE2MdQl1bOkQ + date_accessed: '2024-11-26' publication_year: 2021 - published_by: Google Sheet - license: - name: - url: + published_by: Welfare Footprint + name: Pain hours of hen systems (Welfare Footprint) + description: '' + license: {} is_public: false -wdir: ../../../data/snapshots/fasttrack/latest outs: -- md5: 73b70fa53e689c7115055f982d9b17b5 - size: 228 - path: pain_hours_hen_systems.csv + - md5: daf4b07b53f3eb73c1725266e6e7b83c + size: 466 + path: pain_hours_hen_systems.csv diff --git a/snapshots/fasttrack/latest/plastic_waste_meijer_2021.csv.dvc b/snapshots/fasttrack/latest/plastic_waste_meijer_2021.csv.dvc index 5f13b6b45c4..677696f9254 100644 --- a/snapshots/fasttrack/latest/plastic_waste_meijer_2021.csv.dvc +++ b/snapshots/fasttrack/latest/plastic_waste_meijer_2021.csv.dvc @@ -1,17 +1,17 @@ meta: source: - name: Google Sheet + name: Plastic ocean waste and pollution (Meijer et al. 2021) url: https://www.science.org/doi/10.1126/sciadv.aaz5803 - source_data_url: + source_data_url: |- https://docs.google.com/spreadsheets/d/e/2PACX-1vTD6oDffmZRLtbnYvEY4GKVbNhQO7oZpuivywu0lC_lIS8EIqy67SPnPYHMlW9ByeuQQ1g4lLt0Igl-/pub?output=csv - date_accessed: '2023-09-20' - publication_year: '2021' - published_by: Google Sheet + date_accessed: '2024-11-26' + publication_year: 2021 + published_by: |- + Meijer, L. J., Van Emmerik, T., Van Der Ent, R., Schmidt, C., & Lebreton, L. (2021). More than 1000 rivers account for 80% of global riverine plastic emissions into the ocean. Science Advances, 7(18), eaaz5803. name: Plastic ocean waste and pollution (Meijer et al. 2021) description: '' license: {} -wdir: ../../../data/snapshots/fasttrack/latest outs: -- md5: 9622364b41ad11332ca7164e8da26a20 - size: 12674 - path: plastic_waste_meijer_2021.csv + - md5: cdffe453c306eb9a30aef5dba633f82a + size: 12841 + path: plastic_waste_meijer_2021.csv diff --git a/snapshots/fasttrack/latest/sentience_institute.csv.dvc b/snapshots/fasttrack/latest/sentience_institute.csv.dvc index 96eb93e0cce..121bce8e039 100644 --- a/snapshots/fasttrack/latest/sentience_institute.csv.dvc +++ b/snapshots/fasttrack/latest/sentience_institute.csv.dvc @@ -1,17 +1,16 @@ meta: source: - name: Google Sheet + name: 'Sentience Institute. Animals, Food, and Technology (AFT) Survey: 2021 Update.' url: https://www.sentienceinstitute.org/aft-survey-2021 - source_data_url: + source_data_url: |- https://docs.google.com/spreadsheets/d/e/2PACX-1vTHuvrVzfeof4Srke_7bxUlu6eKNZ7RHrjrP6GD184aqrlIiojtSdGH4Hzu48JNynpiES7CX4nf1kUI/pub?output=csv - date_accessed: '2023-09-18' - publication_year: '2021' - published_by: Google Sheet + date_accessed: '2024-11-26' + publication_year: 2021 + published_by: Sentience Institute name: Survey attitudes to livestock farming (Sentience Institute) description: '' license: {} -wdir: ../../../data/snapshots/fasttrack/latest outs: -- md5: 8d9a1e7de3ccba2495c5bd97cbfc2e62 - size: 1540 - path: sentience_institute.csv + - md5: 8d9a1e7de3ccba2495c5bd97cbfc2e62 + size: 1540 + path: sentience_institute.csv diff --git a/snapshots/fasttrack/latest/treatment_gap_anxiety_disorders_world_mental_health_surveys.csv.dvc b/snapshots/fasttrack/latest/treatment_gap_anxiety_disorders_world_mental_health_surveys.csv.dvc index b5d9ea6ee6a..e41becaeee8 100644 --- a/snapshots/fasttrack/latest/treatment_gap_anxiety_disorders_world_mental_health_surveys.csv.dvc +++ b/snapshots/fasttrack/latest/treatment_gap_anxiety_disorders_world_mental_health_surveys.csv.dvc @@ -1,47 +1,19 @@ meta: - namespace: fasttrack - short_name: treatment_gap_anxiety_disorders_world_mental_health_surveys - file_extension: csv - date_accessed: 2023-05-12 - name: Treatment gap for anxiety disorders - World Mental Health Surveys - Alonso - et al. 2017 - description: 'This dataset comes from the World Mental Health surveys, which conducted - national studies in 21 countries, using validated structured interviews to survey - members of the general population about symptoms of mental illnesses they had - in the past 12 months and their lifetime so far. The source describes the dataset: - "Data came from 24 community epidemiological surveys administered in 21 countries - as part of the WMH surveys (Kessler & Ustun, 2004). These included 12 surveys - carried out in high-income countries, 6 surveys in upper-middle-income countries - and 6 in low or lower-middle income countries (see table 1). The majority of surveys - were based on nationally representative household samples. Three were representative - of urban areas in their countries (Colombia, Mexico, and Peru). Three were representative - of selected regions in their countries (Japan, Nigeria, and Murcia, Spain). Four - were representative of selected Metropolitan Areas (Sao Paulo, Brazil; Medellin, - Colombia; and Beijing-Shanghai and Shenzhen in the People’s Republic of China - (PRC)). Trained lay interviewers conducted face-to-face interviews with respondents, - aged 18 years and over. The interviews took place within the households of the - respondents. To reduce respondent burden, the interview was divided into two parts. - Part I assessed core mental disorders and was administered to all respondents. - Part II, which assessed additional disorders and correlates, was administered - to all Part I respondents who met lifetime criteria for any disorder plus a probability - subsample of other Part I respondents. Part II data, the focus of this report, - were weighted by the inverse of their probabilities of selection into Part II - and additionally weighted to adjust samples to match population distributions - on the cross-classification of key socio-demographic and geographic variables. - Further details about WMH sampling and weighting are available elsewhere(Heeringa - et al., 2008). Response rates ranged between 45.9% and 97.2% and had a weighted - average of 70.1% across all surveys."' - source_name: Google Sheet - url: https://pubmed.ncbi.nlm.nih.gov/29356216/ - source_published_by: Google Sheet - source_data_url: https://docs.google.com/spreadsheets/d/e/2PACX-1vQtKZYhI1TbqVZILdDFZuSYJcew8xyYKc9Euyzfzwz6g8O28Qfapc-QJfVetZqJF2N8mNLItaErz63_/pub?output=csv - license_url: https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6008788/ - license_name: Made freely available by authors - is_public: true - version: latest - publication_year: 2017 -wdir: ../../../data/snapshots/fasttrack/latest + source: + name: Treatment gap for anxiety disorders - World Mental Health Surveys - Alonso et al. 2017 + url: https://pubmed.ncbi.nlm.nih.gov/29356216/ + source_data_url: |- + https://docs.google.com/spreadsheets/d/e/2PACX-1vQtKZYhI1TbqVZILdDFZuSYJcew8xyYKc9Euyzfzwz6g8O28Qfapc-QJfVetZqJF2N8mNLItaErz63_/pub?output=csv + date_accessed: '2024-11-26' + publication_year: 2017 + published_by: Alonso et al. (2017) + name: Treatment gap for anxiety disorders - World Mental Health Surveys - Alonso et al. 2017 + description: |- + This dataset comes from the World Mental Health surveys, which conducted national studies in 21 countries, using validated structured interviews to survey members of the general population about symptoms of mental illnesses they had in the past 12 months and their lifetime so far. The source describes the dataset: "Data came from 24 community epidemiological surveys administered in 21 countries as part of the WMH surveys (Kessler & Ustun, 2004). These included 12 surveys carried out in high-income countries, 6 surveys in upper-middle-income countries and 6 in low or lower-middle income countries (see table 1). The majority of surveys were based on nationally representative household samples. Three were representative of urban areas in their countries (Colombia, Mexico, and Peru). Three were representative of selected regions in their countries (Japan, Nigeria, and Murcia, Spain). Four were representative of selected Metropolitan Areas (Sao Paulo, Brazil; Medellin, Colombia; and Beijing-Shanghai and Shenzhen in the People’s Republic of China (PRC)). Trained lay interviewers conducted face-to-face interviews with respondents, aged 18 years and over. The interviews took place within the households of the respondents. To reduce respondent burden, the interview was divided into two parts. Part I assessed core mental disorders and was administered to all respondents. Part II, which assessed additional disorders and correlates, was administered to all Part I respondents who met lifetime criteria for any disorder plus a probability subsample of other Part I respondents. Part II data, the focus of this report, were weighted by the inverse of their probabilities of selection into Part II and additionally weighted to adjust samples to match population distributions on the cross-classification of key socio-demographic and geographic variables. Further details about WMH sampling and weighting are available elsewhere(Heeringa et al., 2008). Response rates ranged between 45.9% and 97.2% and had a weighted average of 70.1% across all surveys." + license: + name: Made freely available by authors + url: https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6008788/ outs: -- md5: 24b2bc09a23fe8ed5ab7e8a37df42c4d - size: 1297 - path: treatment_gap_anxiety_disorders_world_mental_health_surveys.csv + - md5: 24b2bc09a23fe8ed5ab7e8a37df42c4d + size: 1297 + path: treatment_gap_anxiety_disorders_world_mental_health_surveys.csv diff --git a/snapshots/fasttrack/latest/whm_treatment_gap_anxiety_disorders.csv.dvc b/snapshots/fasttrack/latest/whm_treatment_gap_anxiety_disorders.csv.dvc index cba58543e82..0a3591664e3 100644 --- a/snapshots/fasttrack/latest/whm_treatment_gap_anxiety_disorders.csv.dvc +++ b/snapshots/fasttrack/latest/whm_treatment_gap_anxiety_disorders.csv.dvc @@ -1,47 +1,19 @@ meta: - namespace: fasttrack - short_name: whm_treatment_gap_anxiety_disorders - file_extension: csv - date_accessed: 2023-06-15 + source: + name: Treatment gap for anxiety disorders - World Mental Health Surveys - Alonso et al. 2017 + url: https://pubmed.ncbi.nlm.nih.gov/29356216/ + source_data_url: |- + https://docs.google.com/spreadsheets/d/e/2PACX-1vTQERY6SffT6Lc4ogBdVxjBlPFiOOluxEd3h9oAbrRtSy5YXQ0BTYqZFhrF1wl5N9j6Ko-Mm2XwHZtZ/pub?output=csv + date_accessed: '2024-11-26' + publication_year: 2017 + published_by: Alonso et al. (2017) name: Treatment gap for anxiety disorders (WMH, 2017) - description: 'This dataset comes from the World Mental Health surveys, which conducted - national studies in 21 countries, using validated structured interviews to survey - members of the general population about symptoms of mental illnesses they had - in the past 12 months and their lifetime so far. The source describes the dataset: - "Data came from 24 community epidemiological surveys administered in 21 countries - as part of the WMH surveys (Kessler & Ustun, 2004). These included 12 surveys - carried out in high-income countries, 6 surveys in upper-middle-income countries - and 6 in low or lower-middle income countries (see table 1). The majority of surveys - were based on nationally representative household samples. Three were representative - of urban areas in their countries (Colombia, Mexico, and Peru). Three were representative - of selected regions in their countries (Japan, Nigeria, and Murcia, Spain). Four - were representative of selected Metropolitan Areas (Sao Paulo, Brazil; Medellin, - Colombia; and Beijing-Shanghai and Shenzhen in the People’s Republic of China - (PRC)). Trained lay interviewers conducted face-to-face interviews with respondents, - aged 18 years and over. The interviews took place within the households of the - respondents. To reduce respondent burden, the interview was divided into two parts. - Part I assessed core mental disorders and was administered to all respondents. - Part II, which assessed additional disorders and correlates, was administered - to all Part I respondents who met lifetime criteria for any disorder plus a probability - subsample of other Part I respondents. Part II data, the focus of this report, - were weighted by the inverse of their probabilities of selection into Part II - and additionally weighted to adjust samples to match population distributions - on the cross-classification of key socio-demographic and geographic variables. - Further details about WMH sampling and weighting are available elsewhere(Heeringa - et al., 2008). Response rates ranged between 45.9% and 97.2% and had a weighted - average of 70.1% across all surveys."' - source_name: Google Sheet - url: https://pubmed.ncbi.nlm.nih.gov/29356216/ - source_published_by: Google Sheet - source_data_url: - https://docs.google.com/spreadsheets/d/e/2PACX-1vTQERY6SffT6Lc4ogBdVxjBlPFiOOluxEd3h9oAbrRtSy5YXQ0BTYqZFhrF1wl5N9j6Ko-Mm2XwHZtZ/pub?output=csv - license_url: https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6008788/ - license_name: Made freely available by authors - is_public: true - version: latest - publication_year: 2017 -wdir: ../../../data/snapshots/fasttrack/latest + description: |- + This dataset comes from the World Mental Health surveys, which conducted national studies in 21 countries, using validated structured interviews to survey members of the general population about symptoms of mental illnesses they had in the past 12 months and their lifetime so far. The source describes the dataset: "Data came from 24 community epidemiological surveys administered in 21 countries as part of the WMH surveys (Kessler & Ustun, 2004). These included 12 surveys carried out in high-income countries, 6 surveys in upper-middle-income countries and 6 in low or lower-middle income countries (see table 1). The majority of surveys were based on nationally representative household samples. Three were representative of urban areas in their countries (Colombia, Mexico, and Peru). Three were representative of selected regions in their countries (Japan, Nigeria, and Murcia, Spain). Four were representative of selected Metropolitan Areas (Sao Paulo, Brazil; Medellin, Colombia; and Beijing-Shanghai and Shenzhen in the People’s Republic of China (PRC)). Trained lay interviewers conducted face-to-face interviews with respondents, aged 18 years and over. The interviews took place within the households of the respondents. To reduce respondent burden, the interview was divided into two parts. Part I assessed core mental disorders and was administered to all respondents. Part II, which assessed additional disorders and correlates, was administered to all Part I respondents who met lifetime criteria for any disorder plus a probability subsample of other Part I respondents. Part II data, the focus of this report, were weighted by the inverse of their probabilities of selection into Part II and additionally weighted to adjust samples to match population distributions on the cross-classification of key socio-demographic and geographic variables. Further details about WMH sampling and weighting are available elsewhere(Heeringa et al., 2008). Response rates ranged between 45.9% and 97.2% and had a weighted average of 70.1% across all surveys." + license: + name: Made freely available by authors + url: https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6008788/ outs: -- md5: 2ef040684f17d2121ded34c460efa4aa - size: 2000 - path: whm_treatment_gap_anxiety_disorders.csv + - md5: 2ef040684f17d2121ded34c460efa4aa + size: 2000 + path: whm_treatment_gap_anxiety_disorders.csv diff --git a/snapshots/forests/2024-07-10/dominant_driver.py b/snapshots/forests/2024-07-10/dominant_driver.py index 8271d77cb2f..80469748b50 100644 --- a/snapshots/forests/2024-07-10/dominant_driver.py +++ b/snapshots/forests/2024-07-10/dominant_driver.py @@ -10,7 +10,6 @@ """ - from pathlib import Path import click diff --git a/snapshots/forests/2024-08-07/dominant_driver.py b/snapshots/forests/2024-08-07/dominant_driver.py index fbf275bb0a0..e1365fc8b56 100644 --- a/snapshots/forests/2024-08-07/dominant_driver.py +++ b/snapshots/forests/2024-08-07/dominant_driver.py @@ -2,7 +2,6 @@ Option to get data from Google Earth Engine is in 2024-07-10 version of this script """ - from pathlib import Path import click diff --git a/snapshots/ggdc/2020-10-01/ggdc_maddison.py b/snapshots/ggdc/2020-10-01/ggdc_maddison.py index 7978eacf0e6..fb986488d78 100644 --- a/snapshots/ggdc/2020-10-01/ggdc_maddison.py +++ b/snapshots/ggdc/2020-10-01/ggdc_maddison.py @@ -1,6 +1,4 @@ -"""This script should be manually adapted and executed on the event of an update of the Maddison Project Database. - -""" +"""This script should be manually adapted and executed on the event of an update of the Maddison Project Database.""" import click diff --git a/snapshots/health/latest/global_health_mpox.csv.dvc b/snapshots/health/latest/global_health_mpox.csv.dvc index 571943f8d2e..4d27ab747a3 100644 --- a/snapshots/health/latest/global_health_mpox.csv.dvc +++ b/snapshots/health/latest/global_health_mpox.csv.dvc @@ -22,6 +22,6 @@ meta: url: https://global.health/terms-of-use/ outs: - - md5: 19ced465e9f2dbe57c977dfdcd971ab3 - size: 16620009 + - md5: 08388d2230adafbb7fe28ddcd1eb0dc8 + size: 16813136 path: global_health_mpox.csv diff --git a/snapshots/antibiotics/2024-11-20/bloodstream_amr.py b/snapshots/hmd/2024-11-27/hmd.py similarity index 78% rename from snapshots/antibiotics/2024-11-20/bloodstream_amr.py rename to snapshots/hmd/2024-11-27/hmd.py index 3785f615d9d..c5180a4dfa6 100644 --- a/snapshots/antibiotics/2024-11-20/bloodstream_amr.py +++ b/snapshots/hmd/2024-11-27/hmd.py @@ -12,10 +12,10 @@ @click.command() @click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") -@click.option("--path-to-file", prompt=True, type=str, help="Path to local data file.") +@click.option("--path-to-file", "-f", prompt=True, type=str, help="Path to local data file.") def main(path_to_file: str, upload: bool) -> None: # Create a new snapshot. - snap = Snapshot(f"antibiotics/{SNAPSHOT_VERSION}/bloodstream_amr.csv") + snap = Snapshot(f"hmd/{SNAPSHOT_VERSION}/hmd.zip") # Copy local data file to snapshots data folder, add file to DVC and upload to S3. snap.create_snapshot(filename=path_to_file, upload=upload) diff --git a/snapshots/hmd/2024-11-27/hmd.zip.dvc b/snapshots/hmd/2024-11-27/hmd.zip.dvc new file mode 100644 index 00000000000..63f4dfe3a51 --- /dev/null +++ b/snapshots/hmd/2024-11-27/hmd.zip.dvc @@ -0,0 +1,74 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Human Mortality Database + description: |- + The Human Mortality Database (HMD) contains original calculations of all-cause death rates and life tables for national populations (countries or areas), as well as the input data used in constructing those tables. The input data consist of death counts from vital statistics, plus census counts, birth counts, and population estimates from various sources. + + + # Scope and basic principles + + The database is limited by design to populations where death registration and census data are virtually complete, since this type of information is required for the uniform method used to reconstruct historical data series. As a result, the countries and areas included here are relatively wealthy and for the most part highly industrialized. + + The main goal of the Human Mortality Database is to document the longevity revolution of the modern era and to facilitate research into its causes and consequences. As much as possible, the authors of the database have followed four guiding principles: comparability, flexibility, accessibility, reproducibility. + + + # Computing death rates and life tables + + Their process for computing mortality rates and life tables can be described in terms of six steps, corresponding to six data types that are available from the HMD. Here is an overview of the process: + + 1. Births. Annual counts of live births by sex are collected for each population over the longest possible time period. These counts are used mainly for making population estimates at younger ages. + 2. Deaths. Death counts are collected at the finest level of detail available. If raw data are aggregated, uniform methods are used to estimate death counts by completed age (i.e., age-last-birthday at time of death), calendar year of death, and calendar year of birth. + 3. Population size. Annual estimates of population size on January 1st are either obtained from another source or are derived from census data plus birth and death counts. + 4. Exposure-to-risk. Estimates of the population exposed to the risk of death during some age-time interval are based on annual (January 1st) population estimates, with a small correction that reflects the timing of deaths within the interval. + 5. Death rates. Death rates are always a ratio of the death count for a given age-time interval divided by an estimate of the exposure-to-risk in the same interval. + 6. Life tables. To build a life table, probabilities of death are computed from death rates. These probabilities are used to construct life tables, which include life expectancies and other useful indicators of mortality and longevity. + + + # Corrections to the data + + The data presented here have been corrected for gross errors (e.g., a processing error whereby 3,800 becomes 38,000 in a published statistical table would be obvious in most cases, and it would be corrected). However, the authors have not attempted to correct the data for systematic age misstatement (misreporting of age) or coverage errors (over- or under-enumeration of people or events). + + Some available studies assess the completeness of census coverage or death registration in the various countries, and more work is needed in this area. However, in developing the database thus far, the authors did not consider it feasible or desirable to attempt corrections of this sort, especially since it would be impossible to correct the data by a uniform method across all countries. + + + # Age misreporting + + Populations are included here if there is a well-founded belief that the coverage of their census and vital registration systems is relatively high, and thus, that fruitful analyses by both specialists and non-specialists should be possible with these data. Nevertheless, there is evidence of both age heaping (overreporting ages ending in "0" or "5") and age exaggeration in these data. + + In general, the degree of age heaping in these data varies by the time period and population considered, but it is usually no burden to scientific analysis. In most cases, it is sufficient to analyze data in five-year age groups in order to avoid the false impressions created by this particular form of age misstatement. + + Age exaggeration, on the other hand, is a more insidious problem. The authors' approach is guided by the conventional wisdom that age reporting in death registration systems is typically more reliable than in census counts or official population estimates. For this reason, the authors derive population estimates at older ages from the death counts themselves, employing extinct cohort methods. Such methods eliminate some, but certainly not all, of the biases in old-age mortality estimates due to age exaggeration. + + + # Uniform set of procedures + + A key goal of this project is to follow a uniform set of procedures for each population. This approach does not guarantee the cross-national comparability of the data. Rather, it ensures only that the authors have not introduced biases by the authors' own manipulations. The desire of the authors for uniformity had to face the challenge that raw data come in a variety of formats (for example, 1-year versus 5-year age groups). The authors' general approach to this problem is that the available raw data are used first to estimate two quantities: 1) the number of deaths by completed age, year of birth, and year of death; and 2) population estimates by single years of age on January 1 of each year. For each population, these calculations are performed separately by sex. From these two pieces of information, they compute death rates and life tables in a variety of age-time configurations. + + It is reasonable to ask whether a single procedure is the best method for treating the data from a variety of populations. Here, two points must be considered. First, the authors' uniform methodology is based on procedures that were developed separately, though following similar principles, for various countries and by different researchers. Earlier methods were synthesized by choosing what they considered the best among alternative procedures and by eliminating superficial inconsistencies. The second point is that a uniform procedure is possible only because the authors have not attempted to correct the data for reporting and coverage errors. Although some general principles could be followed, such problems would have to be addressed individually for each population. + + Although the authors adhere strictly to a uniform procedure, the data for each population also receive significant individualized attention. Each country or area is assigned to an individual researcher, who takes responsibility for assembling and checking the data for errors. In addition, the person assigned to each country/area checks the authors' data against other available sources. These procedures help to assure a high level of data quality, but assistance from database users in identifying problems is always appreciated! + date_published: "2024-11-13" + # Citation + producer: Human Mortality Database + citation_full: |- + HMD. Human Mortality Database. Max Planck Institute for Demographic Research (Germany), University of California, Berkeley (USA), and French Institute for Demographic Studies (France). Available at www.mortality.org. + + See also the methods protocol: + Wilmoth, J. R., Andreev, K., Jdanov, D., Glei, D. A., Riffe, T., Boe, C., Bubenheim, M., Philipov, D., Shkolnikov, V., Vachon, P., Winant, C., & Barbieri, M. (2021). Methods protocol for the human mortality database (v6). [Available online](https://www.mortality.org/File/GetDocument/Public/Docs/MethodsProtocolV6.pdf) (needs log in to mortality.org). + attribution_short: HMD + # Files + url_main: https://www.mortality.org/Data/ZippedDataFiles + date_accessed: 2024-11-27 + + # License + license: + name: CC BY 4.0 + url: https://www.mortality.org/Data/UserAgreement + +outs: + - md5: ceed045241a19573e6621423b582558e + size: 147314590 + path: hmd.zip diff --git a/snapshots/hmd/2024-12-01/hmd_country.py b/snapshots/hmd/2024-12-01/hmd_country.py new file mode 100644 index 00000000000..356e913b41d --- /dev/null +++ b/snapshots/hmd/2024-12-01/hmd_country.py @@ -0,0 +1,32 @@ +"""Download data manually: + +- Go to https://mortality.org/Data/ZippedDataFiles +- Scroll down to "By country" section +- Click on "All HMD countries" + +Note: you need to be logged in to download the data. +""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +@click.option("--path-to-file", "-f", prompt=True, type=str, help="Path to local data file.") +def main(path_to_file: str, upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"hmd/{SNAPSHOT_VERSION}/hmd_country.zip") + + # Copy local data file to snapshots data folder, add file to DVC and upload to S3. + snap.create_snapshot(filename=path_to_file, upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/hmd/2024-12-01/hmd_country.zip.dvc b/snapshots/hmd/2024-12-01/hmd_country.zip.dvc new file mode 100644 index 00000000000..9a0d86340f9 --- /dev/null +++ b/snapshots/hmd/2024-12-01/hmd_country.zip.dvc @@ -0,0 +1,79 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Human Mortality Database, by country + description: |- + The Human Mortality Database (HMD) contains original calculations of all-cause death rates and life tables for national populations (countries or areas), as well as the input data used in constructing those tables. The input data consist of death counts from vital statistics, plus census counts, birth counts, and population estimates from various sources. + + + # Scope and basic principles + + The database is limited by design to populations where death registration and census data are virtually complete, since this type of information is required for the uniform method used to reconstruct historical data series. As a result, the countries and areas included here are relatively wealthy and for the most part highly industrialized. + + The main goal of the Human Mortality Database is to document the longevity revolution of the modern era and to facilitate research into its causes and consequences. As much as possible, the authors of the database have followed four guiding principles: comparability, flexibility, accessibility, reproducibility. + + + # Computing death rates and life tables + + Their process for computing mortality rates and life tables can be described in terms of six steps, corresponding to six data types that are available from the HMD. Here is an overview of the process: + + 1. Births. Annual counts of live births by sex are collected for each population over the longest possible time period. These counts are used mainly for making population estimates at younger ages. + 2. Deaths. Death counts are collected at the finest level of detail available. If raw data are aggregated, uniform methods are used to estimate death counts by completed age (i.e., age-last-birthday at time of death), calendar year of death, and calendar year of birth. + 3. Population size. Annual estimates of population size on January 1st are either obtained from another source or are derived from census data plus birth and death counts. + 4. Exposure-to-risk. Estimates of the population exposed to the risk of death during some age-time interval are based on annual (January 1st) population estimates, with a small correction that reflects the timing of deaths within the interval. + 5. Death rates. Death rates are always a ratio of the death count for a given age-time interval divided by an estimate of the exposure-to-risk in the same interval. + 6. Life tables. To build a life table, probabilities of death are computed from death rates. These probabilities are used to construct life tables, which include life expectancies and other useful indicators of mortality and longevity. + + + # Corrections to the data + + The data presented here have been corrected for gross errors (e.g., a processing error whereby 3,800 becomes 38,000 in a published statistical table would be obvious in most cases, and it would be corrected). However, the authors have not attempted to correct the data for systematic age misstatement (misreporting of age) or coverage errors (over- or under-enumeration of people or events). + + Some available studies assess the completeness of census coverage or death registration in the various countries, and more work is needed in this area. However, in developing the database thus far, the authors did not consider it feasible or desirable to attempt corrections of this sort, especially since it would be impossible to correct the data by a uniform method across all countries. + + + # Age misreporting + + Populations are included here if there is a well-founded belief that the coverage of their census and vital registration systems is relatively high, and thus, that fruitful analyses by both specialists and non-specialists should be possible with these data. Nevertheless, there is evidence of both age heaping (overreporting ages ending in "0" or "5") and age exaggeration in these data. + + In general, the degree of age heaping in these data varies by the time period and population considered, but it is usually no burden to scientific analysis. In most cases, it is sufficient to analyze data in five-year age groups in order to avoid the false impressions created by this particular form of age misstatement. + + Age exaggeration, on the other hand, is a more insidious problem. The authors' approach is guided by the conventional wisdom that age reporting in death registration systems is typically more reliable than in census counts or official population estimates. For this reason, the authors derive population estimates at older ages from the death counts themselves, employing extinct cohort methods. Such methods eliminate some, but certainly not all, of the biases in old-age mortality estimates due to age exaggeration. + + + # Uniform set of procedures + + A key goal of this project is to follow a uniform set of procedures for each population. This approach does not guarantee the cross-national comparability of the data. Rather, it ensures only that the authors have not introduced biases by the authors' own manipulations. The desire of the authors for uniformity had to face the challenge that raw data come in a variety of formats (for example, 1-year versus 5-year age groups). The authors' general approach to this problem is that the available raw data are used first to estimate two quantities: 1) the number of deaths by completed age, year of birth, and year of death; and 2) population estimates by single years of age on January 1 of each year. For each population, these calculations are performed separately by sex. From these two pieces of information, they compute death rates and life tables in a variety of age-time configurations. + + It is reasonable to ask whether a single procedure is the best method for treating the data from a variety of populations. Here, two points must be considered. First, the authors' uniform methodology is based on procedures that were developed separately, though following similar principles, for various countries and by different researchers. Earlier methods were synthesized by choosing what they considered the best among alternative procedures and by eliminating superficial inconsistencies. The second point is that a uniform procedure is possible only because the authors have not attempted to correct the data for reporting and coverage errors. Although some general principles could be followed, such problems would have to be addressed individually for each population. + + Although the authors adhere strictly to a uniform procedure, the data for each population also receive significant individualized attention. Each country or area is assigned to an individual researcher, who takes responsibility for assembling and checking the data for errors. In addition, the person assigned to each country/area checks the authors' data against other available sources. These procedures help to assure a high level of data quality, but assistance from database users in identifying problems is always appreciated! + description_snapshot: |- + HMD data by country. This contains the raw data, including their "input data", which HMD defines as: + + The Input Database houses the raw data that are the basis for all HMD calculations. Input data files for each population are accessible from the country page. + + date_published: "2024-11-13" + # Citation + producer: Human Mortality Database + citation_full: |- + HMD. Human Mortality Database. Max Planck Institute for Demographic Research (Germany), University of California, Berkeley (USA), and French Institute for Demographic Studies (France). Available at www.mortality.org. + + See also the methods protocol: + Wilmoth, J. R., Andreev, K., Jdanov, D., Glei, D. A., Riffe, T., Boe, C., Bubenheim, M., Philipov, D., Shkolnikov, V., Vachon, P., Winant, C., & Barbieri, M. (2021). Methods protocol for the human mortality database (v6). [Available online](https://www.mortality.org/File/GetDocument/Public/Docs/MethodsProtocolV6.pdf) (needs log in to mortality.org). + attribution_short: HMD + # Files + url_main: https://www.mortality.org/Data/ZippedDataFiles + date_accessed: 2024-11-27 + + # License + license: + name: CC BY 4.0 + url: https://www.mortality.org/Data/UserAgreement + +outs: + - md5: efae1882e47e8132bd5a2add9f7e445a + size: 345841896 + path: hmd_country.zip diff --git a/snapshots/homicide/2023-01-03/who_mort_db.py b/snapshots/homicide/2023-01-03/who_mort_db.py index 3c28547d1e6..d336074f7c7 100644 --- a/snapshots/homicide/2023-01-03/who_mort_db.py +++ b/snapshots/homicide/2023-01-03/who_mort_db.py @@ -1,6 +1,6 @@ """This script has been autogenerated. - Download full data set from here: https://platform.who.int/mortality/themes/theme-details/topics/indicator-groups/indicator-group-details/MDB/violence - Click on the download button and then select 'full dataset'. It doesn't matter what age groups or years you select it will download them all anyway. +Download full data set from here: https://platform.who.int/mortality/themes/theme-details/topics/indicator-groups/indicator-group-details/MDB/violence +Click on the download button and then select 'full dataset'. It doesn't matter what age groups or years you select it will download them all anyway. """ import pathlib diff --git a/snapshots/homicide/2024-07-30/who_mort_db.py b/snapshots/homicide/2024-07-30/who_mort_db.py index 9a686fa3933..14ef36dc7c0 100644 --- a/snapshots/homicide/2024-07-30/who_mort_db.py +++ b/snapshots/homicide/2024-07-30/who_mort_db.py @@ -1,6 +1,6 @@ """Script to create a snapshot of dataset. - Download full data set from here: https://platform.who.int/mortality/themes/theme-details/topics/indicator-groups/indicator-group-details/MDB/violence - Click on the download button and then select 'full dataset'. It doesn't matter what age groups or years you select it will download them all anyway. +Download full data set from here: https://platform.who.int/mortality/themes/theme-details/topics/indicator-groups/indicator-group-details/MDB/violence +Click on the download button and then select 'full dataset'. It doesn't matter what age groups or years you select it will download them all anyway. """ from pathlib import Path diff --git a/snapshots/iea/2024-11-20/fossil_fuel_subsidies.py b/snapshots/iea/2024-11-20/fossil_fuel_subsidies.py new file mode 100644 index 00000000000..5cb7627fb7c --- /dev/null +++ b/snapshots/iea/2024-11-20/fossil_fuel_subsidies.py @@ -0,0 +1,32 @@ +"""Script to create a snapshot of dataset. + +To obtain the file, you need to log in into the IEA website and download the XLSX file in: +https://www.iea.org/data-and-statistics/data-product/fossil-fuel-subsidies-database#data-sets + +Note that creating an account is free, and this dataset is also free of cost. + +""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +@click.option("--path-to-file", prompt=True, type=str, help="Path to local data file.") +def main(path_to_file: str, upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"iea/{SNAPSHOT_VERSION}/fossil_fuel_subsidies.xlsx") + + # Copy local data file to snapshots data folder, add file to DVC and upload to S3. + snap.create_snapshot(filename=path_to_file, upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/iea/2024-11-20/fossil_fuel_subsidies.xlsx.dvc b/snapshots/iea/2024-11-20/fossil_fuel_subsidies.xlsx.dvc new file mode 100644 index 00000000000..a88b8b6e19d --- /dev/null +++ b/snapshots/iea/2024-11-20/fossil_fuel_subsidies.xlsx.dvc @@ -0,0 +1,29 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Fossil Fuel Subsidies Database + description: |- + Fossil fuel consumption subsidies for selected countries. + date_published: "2024-10-01" + + # Citation + producer: International Energy Agency + citation_full: |- + International Energy Agency - Fossil Fuel Subsidies Database (2024). + attribution_short: IEA + + # Files + url_main: https://www.iea.org/data-and-statistics/data-product/fossil-fuel-subsidies-database + date_accessed: 2024-11-20 + + # License + license: + name: CC BY 4.0 + url: https://www.iea.org/data-and-statistics/data-product/fossil-fuel-subsidies-database + +outs: + - md5: baa1ef5e6f740931575c19082d28f745 + size: 148786 + path: fossil_fuel_subsidies.xlsx diff --git a/snapshots/ihme_gbd/2024-05-20/gbd_drug_risk.py b/snapshots/ihme_gbd/2024-05-20/gbd_drug_risk.py index 4363b21e6a5..2f1012f2a6b 100644 --- a/snapshots/ihme_gbd/2024-05-20/gbd_drug_risk.py +++ b/snapshots/ihme_gbd/2024-05-20/gbd_drug_risk.py @@ -28,6 +28,7 @@ We will download and combine the files in the following script. """ + from pathlib import Path import click diff --git a/snapshots/ihme_gbd/2024-05-20/gbd_mental_health.py b/snapshots/ihme_gbd/2024-05-20/gbd_mental_health.py index 127b442b0c8..850fe3200a4 100644 --- a/snapshots/ihme_gbd/2024-05-20/gbd_mental_health.py +++ b/snapshots/ihme_gbd/2024-05-20/gbd_mental_health.py @@ -18,6 +18,7 @@ We will download and combine the files in the following script. """ + from pathlib import Path import click diff --git a/snapshots/ihme_gbd/2024-05-20/gbd_mental_health_burden.py b/snapshots/ihme_gbd/2024-05-20/gbd_mental_health_burden.py index 510b37bea82..260de4d5764 100644 --- a/snapshots/ihme_gbd/2024-05-20/gbd_mental_health_burden.py +++ b/snapshots/ihme_gbd/2024-05-20/gbd_mental_health_burden.py @@ -18,6 +18,7 @@ We will download and combine the files in the following script. """ + from pathlib import Path import click diff --git a/snapshots/ihme_gbd/2024-05-20/gbd_prevalence.py b/snapshots/ihme_gbd/2024-05-20/gbd_prevalence.py index da6ab0f5cc8..9cc88f92043 100644 --- a/snapshots/ihme_gbd/2024-05-20/gbd_prevalence.py +++ b/snapshots/ihme_gbd/2024-05-20/gbd_prevalence.py @@ -18,6 +18,7 @@ We will download and combine the files in the following script. """ + from pathlib import Path import click diff --git a/snapshots/ihme_gbd/2024-05-20/gbd_risk.py b/snapshots/ihme_gbd/2024-05-20/gbd_risk.py index 4898b4e17da..bb5c50aa8de 100644 --- a/snapshots/ihme_gbd/2024-05-20/gbd_risk.py +++ b/snapshots/ihme_gbd/2024-05-20/gbd_risk.py @@ -31,6 +31,7 @@ We will download and combine the files in the following script. """ + from pathlib import Path import click diff --git a/snapshots/ihme_gbd/2024-05-20/impairments.py b/snapshots/ihme_gbd/2024-05-20/impairments.py index d73d0442231..971b6515096 100644 --- a/snapshots/ihme_gbd/2024-05-20/impairments.py +++ b/snapshots/ihme_gbd/2024-05-20/impairments.py @@ -20,7 +20,6 @@ We will download and combine the files in the following script. """ - from pathlib import Path import click diff --git a/snapshots/ihme_gbd/2024-07-02/gbd_healthy_life_expectancy.py b/snapshots/ihme_gbd/2024-07-02/gbd_healthy_life_expectancy.py index 142863cd919..d04224b0009 100644 --- a/snapshots/ihme_gbd/2024-07-02/gbd_healthy_life_expectancy.py +++ b/snapshots/ihme_gbd/2024-07-02/gbd_healthy_life_expectancy.py @@ -17,6 +17,7 @@ We will download and combine the files in the following script. """ + from pathlib import Path import click diff --git a/snapshots/ihme_gbd/2024-07-02/gbd_life_expectancy.py b/snapshots/ihme_gbd/2024-07-02/gbd_life_expectancy.py index c44aca606f6..1fb268ab984 100644 --- a/snapshots/ihme_gbd/2024-07-02/gbd_life_expectancy.py +++ b/snapshots/ihme_gbd/2024-07-02/gbd_life_expectancy.py @@ -15,6 +15,7 @@ We will download and combine the files in the following script. """ + from pathlib import Path import click diff --git a/snapshots/ivs/2023-11-27/integrated_values_survey.py b/snapshots/ivs/2023-11-27/integrated_values_survey.py index 05ac15eeeaf..5e2b63331f7 100644 --- a/snapshots/ivs/2023-11-27/integrated_values_survey.py +++ b/snapshots/ivs/2023-11-27/integrated_values_survey.py @@ -4,14 +4,14 @@ INSTRUCTIONS - 1. Follow the instructions to construct the IVS file from WVS and EVS microdata here: https://www.worldvaluessurvey.org/WVSEVStrend.jsp - The files required are the WVS and EVS trend files, and the merge syntax file (in our case in Stata). Keep these files in the same folder. - 2. Run the EVS_WVS_Merge_Syntax_stata 4.do file in Stata. This will generate the IVS main dataset. - 3. Run ivs_create_file.do in Stata. It will generate the file ivs.csv - 4. Add snapshot. Currently the command is - python snapshots/ivs/{date}/integrated_values_survey.py --path-to-file snapshots/ivs/{date}/ivs.csv - 5. Delete csv file - 6. Run `etl wvs_trust` + 1. Follow the instructions to construct the IVS file from WVS and EVS microdata here: https://www.worldvaluessurvey.org/WVSEVStrend.jsp + The files required are the WVS and EVS trend files, and the merge syntax file (in our case in Stata). Keep these files in the same folder. + 2. Run the EVS_WVS_Merge_Syntax_stata 4.do file in Stata. This will generate the IVS main dataset. + 3. Run ivs_create_file.do in Stata. It will generate the file ivs.csv + 4. Add snapshot. Currently the command is + python snapshots/ivs/{date}/integrated_values_survey.py --path-to-file snapshots/ivs/{date}/ivs.csv + 5. Delete csv file + 6. Run `etl wvs_trust` """ from pathlib import Path diff --git a/snapshots/lgbt_rights/2024-06-03/equaldex_current.py b/snapshots/lgbt_rights/2024-06-03/equaldex_current.py index 0f5312138ea..0649871a6b9 100644 --- a/snapshots/lgbt_rights/2024-06-03/equaldex_current.py +++ b/snapshots/lgbt_rights/2024-06-03/equaldex_current.py @@ -1,4 +1,4 @@ -"""" +""" " Script to create a snapshot of dataset 'Equaldex dataset - Current'. Check the equaldex_extract.py script for the data extraction and update process. """ diff --git a/snapshots/pew/2024-06-03/extract_data.py b/snapshots/pew/2024-06-03/extract_data.py index 7cf23edf4d6..b5632b77db5 100644 --- a/snapshots/pew/2024-06-03/extract_data.py +++ b/snapshots/pew/2024-06-03/extract_data.py @@ -7,7 +7,6 @@ python snapshots/pew/{version}/same_sex_marriage.py --path-to-file snapshots/pew/{version}/same_sex_marriage.csv """ - from pathlib import Path import pandas as pd diff --git a/snapshots/survey/2023-08-04/latinobarometro_trust.py b/snapshots/survey/2023-08-04/latinobarometro_trust.py index 96851f5e7bc..5c50bbb2001 100644 --- a/snapshots/survey/2023-08-04/latinobarometro_trust.py +++ b/snapshots/survey/2023-08-04/latinobarometro_trust.py @@ -3,14 +3,14 @@ INSTRUCTIONS - 1. Go to the Latinobarómetro portal, https://www.latinobarometro.org/, to the Data (Datos) tab, and download all the survey files in the Stata column. - 2. Extract the dta files from the zip files and keep them in the same folder. - 3. Run latinobarometro.do in Stata from the same folder as the datasets. - 4. It generates a csv file, latinobarometro_trust.csv. Copy it to this folder. - 5. Add snapshot. The command is: - python snapshots/survey/{version}/latinobarometro_trust.py --path-to-file snapshots/survey/{version}/latinobarometro_trust.csv - 6. Delete csv file - 7. Run `etl latinobarometro_trust` + 1. Go to the Latinobarómetro portal, https://www.latinobarometro.org/, to the Data (Datos) tab, and download all the survey files in the Stata column. + 2. Extract the dta files from the zip files and keep them in the same folder. + 3. Run latinobarometro.do in Stata from the same folder as the datasets. + 4. It generates a csv file, latinobarometro_trust.csv. Copy it to this folder. + 5. Add snapshot. The command is: + python snapshots/survey/{version}/latinobarometro_trust.py --path-to-file snapshots/survey/{version}/latinobarometro_trust.csv + 6. Delete csv file + 7. Run `etl latinobarometro_trust` */ diff --git a/snapshots/survey/2023-08-07/afrobarometer_trust.py b/snapshots/survey/2023-08-07/afrobarometer_trust.py index 10bb114289a..4ea005e64b2 100644 --- a/snapshots/survey/2023-08-07/afrobarometer_trust.py +++ b/snapshots/survey/2023-08-07/afrobarometer_trust.py @@ -5,14 +5,14 @@ INSTRUCTIONS - 1. In the Afrobarometer Merged data page (https://www.afrobarometer.org/data/merged-data/), download the Merged Round 8 Data (34 countries) (2022) file. - 2. Copy the file to this directory . Though it is a SPSS file, it can be read by Stata. - 3. Runafrobarometer_trust.do in Stata. If it fails, check the name of the dta file in the first line of the code. - 4. The code generates a csv file called afrobarometer_trust.csv. Copy this file to the snapshots/ess/{version} directory. - 5. Add snapshot. The command is: - python snapshots/survey/{version}/afrobarometer_trust.py --path-to-file snapshots/survey/{version}/afrobarometer_trust.csv - 6. Delete csv file (and sav file) - 7. Run `etl afrobarometer_trust` + 1. In the Afrobarometer Merged data page (https://www.afrobarometer.org/data/merged-data/), download the Merged Round 8 Data (34 countries) (2022) file. + 2. Copy the file to this directory . Though it is a SPSS file, it can be read by Stata. + 3. Runafrobarometer_trust.do in Stata. If it fails, check the name of the dta file in the first line of the code. + 4. The code generates a csv file called afrobarometer_trust.csv. Copy this file to the snapshots/ess/{version} directory. + 5. Add snapshot. The command is: + python snapshots/survey/{version}/afrobarometer_trust.py --path-to-file snapshots/survey/{version}/afrobarometer_trust.csv + 6. Delete csv file (and sav file) + 7. Run `etl afrobarometer_trust` */ diff --git a/snapshots/un/2023-01-24/un_sdg.py b/snapshots/un/2023-01-24/un_sdg.py index e9c036e4174..251da4b7e0d 100644 --- a/snapshots/un/2023-01-24/un_sdg.py +++ b/snapshots/un/2023-01-24/un_sdg.py @@ -1,7 +1,8 @@ """Script to create a snapshot of dataset 'United Nations Sustainable Development Goals (2023)'. - As well as a snapshot of the data we collect a snapshot of the dimensions and attributes of the data. - These often change as the dataset contains many different variables with many different dimensions and values/attributes. +As well as a snapshot of the data we collect a snapshot of the dimensions and attributes of the data. +These often change as the dataset contains many different variables with many different dimensions and values/attributes. """ + import datetime as dt import json import os diff --git a/snapshots/un/2023-08-16/un_sdg.py b/snapshots/un/2023-08-16/un_sdg.py index 239af43598a..589280fbbd2 100644 --- a/snapshots/un/2023-08-16/un_sdg.py +++ b/snapshots/un/2023-08-16/un_sdg.py @@ -1,7 +1,8 @@ """Script to create a snapshot of dataset 'United Nations Sustainable Development Goals (2023)'. - As well as a snapshot of the data we collect a snapshot of the dimensions and attributes of the data. - These often change as the dataset contains many different variables with many different dimensions and values/attributes. +As well as a snapshot of the data we collect a snapshot of the dimensions and attributes of the data. +These often change as the dataset contains many different variables with many different dimensions and values/attributes. """ + import datetime as dt import json import os diff --git a/snapshots/un/2024-01-17/urban_agglomerations_300k.py b/snapshots/un/2024-01-17/urban_agglomerations_300k.py index a58c78a2175..c638546cc58 100644 --- a/snapshots/un/2024-01-17/urban_agglomerations_300k.py +++ b/snapshots/un/2024-01-17/urban_agglomerations_300k.py @@ -4,6 +4,7 @@ Each file is processed by cleaning the data, excluding certain columns, and reshaping the data into a format that is easier to analyze. The data from each file is merged into a single DataFrame. This is done by merging on the country, urban agglomeration, latitude, and year. """ + import os from pathlib import Path diff --git a/snapshots/un/2024-08-27/un_sdg.py b/snapshots/un/2024-08-27/un_sdg.py index 2d52a73e5e6..3f2e1c1d9e1 100644 --- a/snapshots/un/2024-08-27/un_sdg.py +++ b/snapshots/un/2024-08-27/un_sdg.py @@ -1,7 +1,8 @@ """Script to create a snapshot of dataset 'United Nations Sustainable Development Goals (2024)'. - As well as a snapshot of the data we collect a snapshot of the dimensions and attributes of the data. - These often change as the dataset contains many different variables with many different dimensions and values/attributes. +As well as a snapshot of the data we collect a snapshot of the dimensions and attributes of the data. +These often change as the dataset contains many different variables with many different dimensions and values/attributes. """ + import datetime as dt import json import os diff --git a/snapshots/un/2024-12-02/un_wpp_lt.py b/snapshots/un/2024-12-02/un_wpp_lt.py new file mode 100644 index 00000000000..41b239039cc --- /dev/null +++ b/snapshots/un/2024-12-02/un_wpp_lt.py @@ -0,0 +1,47 @@ +"""Script to create a snapshot of dataset. + +While this ingest script could be done automatically, as of today (2023-10-02) downloading the files from UN WPP is extremely slow. Hence, I've decided to first manually download these and then run the snapshot ingest script. + +To download this files: + + 1. Go to the CSV Format section of UN WPP page: https://population.un.org/wpp/Download/Standard/CSV/ + 2. Download the Life Tables ZIP files with the estimates (1950-2021): + - https://population.un.org/wpp/Download/Files/1_Indicator%20(Standard)/CSV_FILES/WPP2024_Life_Table_Complete_Medium_Both_1950-2023.csv.gz + - https://population.un.org/wpp/Download/Files/1_Indicator%20(Standard)/CSV_FILES/WPP2024_Life_Table_Complete_Medium_Female_1950-2023.csv.gz + - https://population.un.org/wpp/Download/Files/1_Indicator%20(Standard)/CSV_FILES/WPP2024_Life_Table_Complete_Medium_Male_1950-2023.csv.gz + 3. Run the snapshot script and wait for it to be ingested into S3: + python snapshots/un/2023-10-02/un_wpp_lt.py --path-to-file-all /path/WPP2022_Life_Table_Complete_Medium_Both_1950-2021.zip --path-to-file-f path/WPP2022_Life_Table_Complete_Medium_Female_1950-2021.zip --path-to-file-m path/WPP2022_Life_Table_Complete_Medium_Male_1950-2021.zip + +""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +@click.option("--path-to-file-all", prompt=True, type=str, help="Path to local data file (both sexes).") +@click.option("--path-to-file-f", prompt=True, type=str, help="Path to local data file (female).") +@click.option("--path-to-file-m", prompt=True, type=str, help="Path to local data file (male).") +def main(path_to_file_all: str, path_to_file_f: str, path_to_file_m: str, upload: bool) -> None: + snaps = [ + ("un_wpp_lt_all", path_to_file_all), # ALL + ("un_wpp_lt_f", path_to_file_f), # FEMALE + ("un_wpp_lt_m", path_to_file_m), # MALE + ] + + for snap_props in snaps: + # Create a new snapshot. + snap = Snapshot(f"un/{SNAPSHOT_VERSION}/{snap_props[0]}.csv") + # Copy local data file to snapshots data folder, add file to DVC and upload to S3. + snap.create_snapshot(filename=snap_props[1], upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/un/2024-12-02/un_wpp_lt_all.csv.dvc b/snapshots/un/2024-12-02/un_wpp_lt_all.csv.dvc new file mode 100644 index 00000000000..cfae7bef621 --- /dev/null +++ b/snapshots/un/2024-12-02/un_wpp_lt_all.csv.dvc @@ -0,0 +1,36 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/origin/ +meta: + origin: + # Data product / Snapshot + title: World Population Prospects + description: |- + The World Population Prospects 2024 is the 28th edition of the official estimates and projections of the global population published by the United Nations since 1951. The estimates are based on all available sources of data on population size and levels of fertility, mortality, and international migration for 237 countries or areas. + + For each revision, any new, recent, and historical, information that has become available from population censuses, vital registration of births and deaths, and household surveys is considered to produce consistent time series of population estimates for each country or areas from 1950 to today + + For the estimation period between 1950 and 2023, data from 1,910 censuses were considered in the present evaluation, which is 79 more than the 2022 revision. In some countries, population registers based on administrative data systems provide the necessary information. Population data from censuses or registers referring to 2019 or later were available for 114 countries or areas, representing 48 per cent of the 237 countries or areas included in this analysis (and 54 per cent of the world population). For 43 countries or areas, the most recent available population count was from the period 2014-2018, and for another 57 locations from the period 2009-2013. For the remaining 23 countries or areas, the most recent available census data were from before 2009, that is more than 15 years ago. + date_published: 2024-07-11 + title_snapshot: World Population Prospects - Life Tables (Both sexes) + description_snapshot: |- + Provides single-age life tables up to age 100 for both sexes with a set of values showing the mortality experience of a hypothetical group of infants born at the same time and subject throughout their lifetime to the specific mortality rates of a given year. The following series are provided: age-specific mortality rates (mx), probabilities of dying (qx), probabilities of surviving (px), number surviving (lx), number dying (dx), number of person-years lived (Lx), survivorship ratios (Sx), cumulative stationary population (Tx), average remaining life expectancy (ex) and average number of years lived (ax). + + # Citation + producer: United Nations + citation_full: |- + United Nations, Department of Economic and Social Affairs, Population Division (2024). World Population Prospects 2024, Online Edition. + attribution: UN, World Population Prospects (2024) + attribution_short: UN WPP + + # Files + url_main: https://population.un.org/wpp/Download/ + date_accessed: 2024-12-02 + + # License + license: + name: CC BY 3.0 IGO + url: https://population.un.org/wpp/Download/Standard/MostUsed/ +outs: + - md5: 8efb8acd80396c280c61e8de2cd94fb6 + size: 200036668 + path: un_wpp_lt_all.csv diff --git a/snapshots/un/2024-12-02/un_wpp_lt_f.csv.dvc b/snapshots/un/2024-12-02/un_wpp_lt_f.csv.dvc new file mode 100644 index 00000000000..315bc32d7a3 --- /dev/null +++ b/snapshots/un/2024-12-02/un_wpp_lt_f.csv.dvc @@ -0,0 +1,36 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/origin/ +meta: + origin: + # Data product / Snapshot + title: World Population Prospects + description: |- + The World Population Prospects 2024 is the 28th edition of the official estimates and projections of the global population published by the United Nations since 1951. The estimates are based on all available sources of data on population size and levels of fertility, mortality, and international migration for 237 countries or areas. + + For each revision, any new, recent, and historical, information that has become available from population censuses, vital registration of births and deaths, and household surveys is considered to produce consistent time series of population estimates for each country or areas from 1950 to today + + For the estimation period between 1950 and 2023, data from 1,910 censuses were considered in the present evaluation, which is 79 more than the 2022 revision. In some countries, population registers based on administrative data systems provide the necessary information. Population data from censuses or registers referring to 2019 or later were available for 114 countries or areas, representing 48 per cent of the 237 countries or areas included in this analysis (and 54 per cent of the world population). For 43 countries or areas, the most recent available population count was from the period 2014-2018, and for another 57 locations from the period 2009-2013. For the remaining 23 countries or areas, the most recent available census data were from before 2009, that is more than 15 years ago. + date_published: 2024-07-11 + title_snapshot: World Population Prospects - Life Tables (Female) + description_snapshot: |- + Provides single-age life tables up to age 100 for females with a set of values showing the mortality experience of a hypothetical group of infants born at the same time and subject throughout their lifetime to the specific mortality rates of a given year. The following series are provided: age-specific mortality rates (mx), probabilities of dying (qx), probabilities of surviving (px), number surviving (lx), number dying (dx), number of person-years lived (Lx), survivorship ratios (Sx), cumulative stationary population (Tx), average remaining life expectancy (ex) and average number of years lived (ax). + + # Citation + producer: United Nations + citation_full: |- + United Nations, Department of Economic and Social Affairs, Population Division (2024). World Population Prospects 2024, Online Edition. + attribution: UN, World Population Prospects (2024) + attribution_short: UN WPP + + # Files + url_main: https://population.un.org/wpp/Download/ + date_accessed: 2024-12-02 + + # License + license: + name: CC BY 3.0 IGO + url: https://population.un.org/wpp/Download/Standard/MostUsed/ +outs: + - md5: bc8637e9e160cc99e496e245f1d4bff6 + size: 199164889 + path: un_wpp_lt_f.csv diff --git a/snapshots/un/2024-12-02/un_wpp_lt_m.csv.dvc b/snapshots/un/2024-12-02/un_wpp_lt_m.csv.dvc new file mode 100644 index 00000000000..f4e272590a4 --- /dev/null +++ b/snapshots/un/2024-12-02/un_wpp_lt_m.csv.dvc @@ -0,0 +1,36 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/origin/ +meta: + origin: + # Data product / Snapshot + title: World Population Prospects + description: |- + The World Population Prospects 2024 is the 28th edition of the official estimates and projections of the global population published by the United Nations since 1951. The estimates are based on all available sources of data on population size and levels of fertility, mortality, and international migration for 237 countries or areas. + + For each revision, any new, recent, and historical, information that has become available from population censuses, vital registration of births and deaths, and household surveys is considered to produce consistent time series of population estimates for each country or areas from 1950 to today + + For the estimation period between 1950 and 2023, data from 1,910 censuses were considered in the present evaluation, which is 79 more than the 2022 revision. In some countries, population registers based on administrative data systems provide the necessary information. Population data from censuses or registers referring to 2019 or later were available for 114 countries or areas, representing 48 per cent of the 237 countries or areas included in this analysis (and 54 per cent of the world population). For 43 countries or areas, the most recent available population count was from the period 2014-2018, and for another 57 locations from the period 2009-2013. For the remaining 23 countries or areas, the most recent available census data were from before 2009, that is more than 15 years ago. + date_published: 2024-07-11 + title_snapshot: World Population Prospects - Life Tables (Male) + description_snapshot: |- + Provides single-age life tables up to age 100 for males with a set of values showing the mortality experience of a hypothetical group of infants born at the same time and subject throughout their lifetime to the specific mortality rates of a given year. The following series are provided: age-specific mortality rates (mx), probabilities of dying (qx), probabilities of surviving (px), number surviving (lx), number dying (dx), number of person-years lived (Lx), survivorship ratios (Sx), cumulative stationary population (Tx), average remaining life expectancy (ex) and average number of years lived (ax). + + # Citation + producer: United Nations + citation_full: |- + United Nations, Department of Economic and Social Affairs, Population Division (2024). World Population Prospects 2024, Online Edition. + attribution: UN, World Population Prospects (2024) + attribution_short: UN WPP + + # Files + url_main: https://population.un.org/wpp/Download/ + date_accessed: 2024-12-02 + + # License + license: + name: CC BY 3.0 IGO + url: https://population.un.org/wpp/Download/Standard/MostUsed/ +outs: + - md5: da77cb47298654d160e523568d443b29 + size: 199442259 + path: un_wpp_lt_m.csv diff --git a/snapshots/urbanization/2024-10-14/ghsl_degree_of_urbanisation.xlsx.dvc b/snapshots/urbanization/2024-10-14/ghsl_degree_of_urbanisation.xlsx.dvc index 1701120fca4..9d606d204d9 100644 --- a/snapshots/urbanization/2024-10-14/ghsl_degree_of_urbanisation.xlsx.dvc +++ b/snapshots/urbanization/2024-10-14/ghsl_degree_of_urbanisation.xlsx.dvc @@ -11,10 +11,10 @@ meta: # Citation producer: European Commission, Joint Research Centre (JRC) citation_full: |- - Carioli A., Schiavina M., Melchiorri M. (2024): GHS-COUNTRY-STATS R2024A - GHSL Country Statistics by Degree of Urbanization, multitemporal (1975-2030). European Commission, Joint Research Centre (JRC) [Dataset] doi:10.2905/341c0608-5ca5-4ddb-b068-a412e35a3326 PID: http://data.europa.eu/89h/341c0608-5ca5-4ddb-b068-a412e35a3326 + Carioli, Alessandra; Schiavina, Marcello; Melchiorri, Michele (2024): GHS-COUNTRY-STATS R2024A - GHSL Country Statistics by Degree of Urbanization, multitemporal (1975-2030). European Commission, Joint Research Centre (JRC) [Dataset] doi: 10.2905/341c0608-5ca5-4ddb-b068-a412e35a3326 PID: http://data.europa.eu/89h/341c0608-5ca5-4ddb-b068-a412e35a3326 # Files - url_main: https://ghsl.jrc.ec.europa.eu/CFS.php + url_main: https://data.jrc.ec.europa.eu/dataset/341c0608-5ca5-4ddb-b068-a412e35a3326 date_accessed: 2024-10-14 # License diff --git a/snapshots/urbanization/2024-12-02/ghsl_urban_centers.py b/snapshots/urbanization/2024-12-02/ghsl_urban_centers.py new file mode 100644 index 00000000000..4c0ede9b1f5 --- /dev/null +++ b/snapshots/urbanization/2024-12-02/ghsl_urban_centers.py @@ -0,0 +1,25 @@ +"""Script to create a snapshot of dataset. This version of the dataset was provided directly by the source via email (DIJKSTRA Lewis ).""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +@click.option("--path-to-file", prompt=True, type=str, help="Path to local data file.") +def main(path_to_file: str, upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"urbanization/{SNAPSHOT_VERSION}/ghsl_urban_centers.xlsx") + + # Copy local data file to snapshots data folder, add file to DVC and upload to S3. + snap.create_snapshot(filename=path_to_file, upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/urbanization/2024-12-02/ghsl_urban_centers.xlsx.dvc b/snapshots/urbanization/2024-12-02/ghsl_urban_centers.xlsx.dvc new file mode 100644 index 00000000000..f2c69bd3b2c --- /dev/null +++ b/snapshots/urbanization/2024-12-02/ghsl_urban_centers.xlsx.dvc @@ -0,0 +1,33 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/origin/ +meta: + origin: + # Data product / Snapshot + title: Global Human Settlement Layer Dataset - Stats in the City Database + description: |- + The "Stats in the City Database" offers harmonized data on population and population density for 11,422 urban centres. + + This data, based on the Global Human Settlement Layer Dataset, uses the Degree of Urbanisation framework to delineate spatial entities and integrates geospatial data from a variety of open-source datasets. It represents one of the most comprehensive resources for understanding urban population patterns and densities worldwide + date_published: "2024" + + # Citation + producer: European Commission, Joint Research Centre (JRC) + citation_full: |- + Center For International Earth Science Information Network-CIESIN-Columbia University. 2018. “Gridded Population of the World, Version 4 (GPWv4): Population Count, Revision 11.” Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H4JW8BX5 + Pesaresi M., Politis P. (2023): GHS-BUILT-V R2023A - GHS built-up volume grids derived from joint assessment of Sentinel2, Landsat, and global DEM data, multitemporal (1975-2030).European Commission, Joint Research Centre (JRC) PID: http://data.europa.eu/89h/ab2f107a-03cd-47a3-85e5-139d8ec63283, doi:10.2905/AB2F107A-03CD-47A3-85E5-139D8EC63283 + Pesaresi M., Politis P. (2023): GHS-BUILT-S R2023A - GHS built-up surface grid, derived from Sentinel2 composite and Landsat, multitemporal (1975-2030)European Commission, Joint Research Centre (JRC) PID: http://data.europa.eu/89h/9f06f36f-4b11-47ec-abb0-4f8b7b1d72ea, doi:10.2905/9F06F36F-4B11-47EC-ABB0-4F8B7B1D72EA + Schiavina M., Freire S., Carioli A., MacManus K. (2023): GHS-POP R2023A - GHS population grid multitemporal (1975-2030).European Commission, Joint Research Centre (JRC) PID: http://data.europa.eu/89h/2ff68a52-5b5b-4a22-8f40-c41da8332cfe, doi:10.2905/2FF68A52-5B5B-4A22-8F40-C41DA8332CFE + Schiavina M., Freire S., Carioli A., MacManus K. (2023): GHS-POP R2023A - GHS population grid multitemporal (1975-2030).European Commission, Joint Research Centre (JRC) PID: http://data.europa.eu/89h/2ff68a52-5b5b-4a22-8f40-c41da8332cfe, doi:10.2905/2FF68A52-5B5B-4A22-8F40-C41DA8332CFE + Schiavina M., Melchiorri M., Pesaresi M. (2023): GHS-SMOD R2023A - GHS settlement layers, application of the Degree of Urbanisation methodology (stage I) to GHS-POP R2023A and GHS-BUILT-S R2023A, multitemporal (1975-2030)European Commission, Joint Research Centre (JRC) PID: http://data.europa.eu/89h/a0df7a6f-49de-46ea-9bde-563437a6e2ba, doi:10.2905/A0DF7A6F-49DE-46EA-9BDE-563437A6E2BA + + url_main: https://human-settlement.emergency.copernicus.eu/ghs_ucdb_2024.php + date_accessed: 2024-12-02 + + # License + license: + name: CC BY 4.0 + url: https://commission.europa.eu/legal-notice_en +outs: + - md5: 78dbc4fc3cbcbe24cd51fe4f884319e2 + size: 2963003 + path: ghsl_urban_centers.xlsx diff --git a/snapshots/usgs/2024-07-15/historical_statistics_for_mineral_and_material_commodities.py b/snapshots/usgs/2024-07-15/historical_statistics_for_mineral_and_material_commodities.py index 911e9df6b16..3584244c9d7 100644 --- a/snapshots/usgs/2024-07-15/historical_statistics_for_mineral_and_material_commodities.py +++ b/snapshots/usgs/2024-07-15/historical_statistics_for_mineral_and_material_commodities.py @@ -32,26 +32,24 @@ def get_table_of_commodities_and_urls(url_main: str) -> pd.DataFrame: # Extract the rows data = [] - for row in table.find_all("tr")[2:]: # type: ignore + for row in table.find_all("tr")[2:]: cols = row.find_all("td") - if len(cols) >= 6: # Ensure there are enough columns - commodity = cols[0].get_text(strip=True) - supply_demand_url = cols[1].find("a")["href"] if cols[1].find("a") else "NA" - supply_demand_year_update = cols[2].get_text(strip=True) - end_use_url = cols[4].find("a")["href"] if cols[4].find("a") else "NA" - end_use_year_update = cols[5].get_text(strip=True) - - # Add row to data if urls are not 'NA' - if not supply_demand_url.lower().endswith(".pdf") and not end_use_url.lower().endswith(".pdf"): - data.append( - [ - commodity, - supply_demand_url if not supply_demand_url.lower().endswith(".pdf") else "NA", - supply_demand_year_update, - end_use_url if not end_use_url.lower().endswith(".pdf") else "NA", - end_use_year_update, - ] - ) + commodity = cols[0].get_text(strip=True) if len(cols) > 0 else "NA" + supply_demand_url = cols[1].find("a")["href"] if len(cols) > 1 and cols[1].find("a") else "NA" + supply_demand_year_update = cols[2].get_text(strip=True) if len(cols) > 2 else "NA" + end_use_url = cols[4].find("a")["href"] if len(cols) > 4 and cols[4].find("a") else "NA" + end_use_year_update = cols[5].get_text(strip=True) if len(cols) > 5 else "NA" + + # Add row to data regardless of whether URLs are present + data.append( + [ + commodity, + supply_demand_url, + supply_demand_year_update, + end_use_url, + end_use_year_update, + ] + ) # Create a DataFrame with the fetched data. df = pd.DataFrame( @@ -105,15 +103,6 @@ def _fetch_file_url_from_media_path(media_path: str) -> Optional[str]: return data_file_url # type: ignore -def _download_file(url: str, dest_folder: Path, commodity: str) -> None: - response = requests.get(url) - file_path = dest_folder / f"{underscore(commodity)}.xlsx" - file_path.write_bytes(response.content) - - # Wait before sending next query. - sleep(TIME_BETWEEN_QUERIES) - - def download_all_files(df: pd.DataFrame, snapshot_path: Path) -> None: # Ensure the output folder exists. snapshot_path.parent.mkdir(exist_ok=True, parents=True) @@ -129,7 +118,7 @@ def download_all_files(df: pd.DataFrame, snapshot_path: Path) -> None: end_use_dir.mkdir(parents=True, exist_ok=True) # Download files for all commodities. - for index, row in tqdm(df.iterrows(), total=df.shape[0], desc="Downloading files"): + for _, row in tqdm(df.iterrows(), total=df.shape[0], desc="Downloading files"): if row["supply_demand_url"] != "NA": download_file_from_url( url=row["supply_demand_url"], @@ -167,11 +156,12 @@ def main(upload: bool) -> None: # NOTE: This may take a few minutes. df = get_table_of_commodities_and_urls(url_main=snap.metadata.origin.url_main) # type: ignore - # Download the supply-demand statistics file and end-use statistics file for each commodity. + # Download the supply-demand statistics file and end-use statistics file for each commodity in a temporary folder. + # A compressed file will be created at the end in the corresponding data/snapshots/ folder. # NOTE: This may take a few minutes. download_all_files(df=df, snapshot_path=snap.path) - # Upload file to S3. + # Upload zip file to S3. snap.create_snapshot(upload=upload, filename=snap.path) diff --git a/snapshots/usgs/2024-07-15/historical_statistics_for_mineral_and_material_commodities.zip.dvc b/snapshots/usgs/2024-07-15/historical_statistics_for_mineral_and_material_commodities.zip.dvc index e0c6e244f9d..8a8d7286703 100644 --- a/snapshots/usgs/2024-07-15/historical_statistics_for_mineral_and_material_commodities.zip.dvc +++ b/snapshots/usgs/2024-07-15/historical_statistics_for_mineral_and_material_commodities.zip.dvc @@ -28,6 +28,6 @@ meta: url: https://www.usgs.gov/centers/national-minerals-information-center/historical-statistics-mineral-and-material-commodities outs: - - md5: bae42fc20438313cb7dd37a93682f6e6 - size: 7838597 + - md5: 13a7622cbcb1e671075622d8c022b295 + size: 8750002 path: historical_statistics_for_mineral_and_material_commodities.zip diff --git a/snapshots/wb/2024-01-17/pip_api.py b/snapshots/wb/2024-01-17/pip_api.py index 1ef5c1a4a7f..3121e4b4a54 100644 --- a/snapshots/wb/2024-01-17/pip_api.py +++ b/snapshots/wb/2024-01-17/pip_api.py @@ -29,7 +29,6 @@ """ - import io import time from multiprocessing.pool import ThreadPool diff --git a/snapshots/wb/2024-03-27/pip_api.py b/snapshots/wb/2024-03-27/pip_api.py index 48eeb366e3d..a507a778517 100644 --- a/snapshots/wb/2024-03-27/pip_api.py +++ b/snapshots/wb/2024-03-27/pip_api.py @@ -38,7 +38,6 @@ """ - import io import time from multiprocessing.pool import ThreadPool diff --git a/snapshots/wb/2024-06-18/edstats_metadata.py b/snapshots/wb/2024-06-18/edstats_metadata.py index 2c93b710ad3..1ac6fa433ee 100644 --- a/snapshots/wb/2024-06-18/edstats_metadata.py +++ b/snapshots/wb/2024-06-18/edstats_metadata.py @@ -6,7 +6,6 @@ - on the left click on Country -> select World, series -> select All and Time -> select All --> 'Click Download Options' -> Metadata" """ - from pathlib import Path import click diff --git a/snapshots/wb/2024-06-26/extract_projections_from_article.py b/snapshots/wb/2024-06-26/extract_projections_from_article.py index 4b6d3f94ea1..d7d997f6c2d 100644 --- a/snapshots/wb/2024-06-26/extract_projections_from_article.py +++ b/snapshots/wb/2024-06-26/extract_projections_from_article.py @@ -16,7 +16,6 @@ Based on https://stackoverflow.com/questions/62031809/extracting-javascript-variables-into-python-dictionaries """ - import json from pathlib import Path diff --git a/snapshots/wb/2024-10-07/pip_api.py b/snapshots/wb/2024-10-07/pip_api.py index 0d494037556..d791bf23c52 100644 --- a/snapshots/wb/2024-10-07/pip_api.py +++ b/snapshots/wb/2024-10-07/pip_api.py @@ -44,7 +44,6 @@ """ - import io import time from multiprocessing.pool import ThreadPool diff --git a/snapshots/wb/2024-12-03/reproducibility_package_poverty_prosperity_planet.py b/snapshots/wb/2024-12-03/reproducibility_package_poverty_prosperity_planet.py new file mode 100644 index 00000000000..0fa8bed05b3 --- /dev/null +++ b/snapshots/wb/2024-12-03/reproducibility_package_poverty_prosperity_planet.py @@ -0,0 +1,24 @@ +"""Script to create a snapshot of dataset.""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +def main(upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"wb/{SNAPSHOT_VERSION}/reproducibility_package_poverty_prosperity_planet.zip") + + # Download data from source, add file to DVC and upload to S3. + snap.create_snapshot(upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/wb/2024-12-03/reproducibility_package_poverty_prosperity_planet.zip.dvc b/snapshots/wb/2024-12-03/reproducibility_package_poverty_prosperity_planet.zip.dvc new file mode 100644 index 00000000000..08689d46ba7 --- /dev/null +++ b/snapshots/wb/2024-12-03/reproducibility_package_poverty_prosperity_planet.zip.dvc @@ -0,0 +1,30 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Reproducibility package for Poverty, Prosperity and Planet Report 2024 + description: |- + The World Bank has set a clear mission: ending extreme poverty and boosting shared prosperity on a livable planet. This new edition of the biennial series, previously titled Poverty and Shared Prosperity, assesses the three components of the mission and emphasizes that reducing poverty and increasing shared prosperity must be achieved without high costs to the environment. The current polycrisis—where the multiple crises of slow economic growth, increased fragility, climate risks, and heightened uncertainty have come together at the same time—makes national development strategies and international cooperation difficult. This overview summarizes the progress toward achieving these goals, outlines promising pathways to speed up the progress on multiple fronts, and proposes priorities tailored to countries at various levels of poverty, income, and environmental vulnerability. Offering the first post-COVID-19 (Coronavirus) pandemic assessment of global progress on this interlinked agenda, the report finds that global poverty reduction has resumed but at a pace slower than before the COVID-19 crisis. It also provides evidence that the number of countries with high levels of income inequality has declined considerably during the past two decades, but the pace of improvements in shared prosperity has slowed and that inequality remains high in Latin America and the Caribbean and in Sub-Saharan Africa. The report also finds evidence of countries’ increasing ability to manage natural hazards where there has been progress in poverty reduction and shared prosperity; but in the poorest settings, the report finds that climate risks are significantly higher. + date_published: "2024-09-26" + + # Citation + producer: Lakner et al. + citation_full: |- + Lakner, C., Genoni, M. E., Stemmler, H., Yonzan, N., & Tetteh Baah, S. K. (2024). Reproducibility package for Poverty, Prosperity and Planet Report 2024. World Bank. https://doi.org/10.60572/KGE4-CX54 + attribution: Lakner et al. (2024). Reproducibility package for Poverty, Prosperity and Planet Report 2024 + + # Files + url_main: https://reproducibility.worldbank.org/index.php/catalog/189/ + url_download: https://reproducibility.worldbank.org/index.php/catalog/189/download/552/FR_WLD_2024_198.zip + date_accessed: 2024-12-03 + + # License + license: + name: Modified BSD3 + url: https://reproducibility.worldbank.org/index.php/catalog/189/#project_desc_container1674844764972 + +outs: + - md5: 3a942e2ccc863f67d4879250d7468e57 + size: 91787338 + path: reproducibility_package_poverty_prosperity_planet.zip diff --git a/snapshots/who/2023-08-01/mortality_database.py b/snapshots/who/2023-08-01/mortality_database.py index 2f2fdea2a9e..e7be652f9fc 100644 --- a/snapshots/who/2023-08-01/mortality_database.py +++ b/snapshots/who/2023-08-01/mortality_database.py @@ -17,6 +17,7 @@ ICD 10 codes were gathered from the downloaded files and the broad cause group was inferred from the platform. """ + import os from pathlib import Path diff --git a/snapshots/who/2024-07-26/mortality_database.py b/snapshots/who/2024-07-26/mortality_database.py index e72582eb440..f455a20607a 100644 --- a/snapshots/who/2024-07-26/mortality_database.py +++ b/snapshots/who/2024-07-26/mortality_database.py @@ -17,6 +17,7 @@ ICD 10 codes were gathered from the downloaded files and the broad cause group was inferred from the platform. """ + import os from pathlib import Path diff --git a/snapshots/who/2024-08-06/mortality_database_cancer.py b/snapshots/who/2024-08-06/mortality_database_cancer.py index b5d18773ff8..f7a32af5182 100644 --- a/snapshots/who/2024-08-06/mortality_database_cancer.py +++ b/snapshots/who/2024-08-06/mortality_database_cancer.py @@ -17,6 +17,7 @@ ICD 10 codes were gathered from the downloaded files and the broad cause group was inferred from the platform. """ + import os from pathlib import Path diff --git a/snapshots/who/latest/fluid.csv.dvc b/snapshots/who/latest/fluid.csv.dvc index a8a43e6b0d3..9e73e4a37e4 100644 --- a/snapshots/who/latest/fluid.csv.dvc +++ b/snapshots/who/latest/fluid.csv.dvc @@ -16,6 +16,6 @@ meta: The platform accommodates both qualitative and quantitative data which facilitates the tracking of global trends, spread, intensity, and impact of influenza. These data are made freely available to health policy makers in order to assist them in making informed decisions regarding the management of influenza. wdir: ../../../data/snapshots/who/latest outs: - - md5: c7727f835cdd071a329df437f90fb33c - size: 167459700 + - md5: a30bbd904933a05da1e742e84ec9a63b + size: 168149280 path: fluid.csv diff --git a/snapshots/who/latest/flunet.csv.dvc b/snapshots/who/latest/flunet.csv.dvc index b560613ff5b..a36755702fa 100644 --- a/snapshots/who/latest/flunet.csv.dvc +++ b/snapshots/who/latest/flunet.csv.dvc @@ -16,6 +16,6 @@ meta: The data are provided remotely by National Influenza Centres (NICs) of the Global Influenza Surveillance and Response System (GISRS) and other national influenza reference laboratories collaborating actively with GISRS, or are uploaded from WHO regional databases. wdir: ../../../data/snapshots/who/latest outs: - - md5: 3881ea7a7aea573a13496aca6f5e3249 - size: 27145366 + - md5: d3645d43a27ce8aeb1f2ac42f0558694 + size: 27223921 path: flunet.csv diff --git a/tests/data_helpers/test_geo.py b/tests/data_helpers/test_geo.py index 813cec00e7e..da40a5ce0b1 100644 --- a/tests/data_helpers/test_geo.py +++ b/tests/data_helpers/test_geo.py @@ -1,6 +1,4 @@ -"""Test functions in etl.data_helpers.geo module. - -""" +"""Test functions in etl.data_helpers.geo module.""" import json import unittest diff --git a/tests/data_helpers/test_misc.py b/tests/data_helpers/test_misc.py index 149c5ab6b41..64dfce89e1e 100644 --- a/tests/data_helpers/test_misc.py +++ b/tests/data_helpers/test_misc.py @@ -1,6 +1,4 @@ -"""Test functions in etl.data_helpers.misc module. - -""" +"""Test functions in etl.data_helpers.misc module.""" import numpy as np import pandas as pd diff --git a/tests/test_datadiff.py b/tests/test_datadiff.py index 316ecfdeaf1..be4466781c1 100644 --- a/tests/test_datadiff.py +++ b/tests/test_datadiff.py @@ -1,3 +1,6 @@ +import os +from unittest.mock import patch + import pandas as pd from owid.catalog import Dataset, DatasetMeta, Table @@ -19,6 +22,7 @@ def _create_datasets(tmp_path): return ds_a, ds_b +@patch.dict(os.environ, {"OWID_STRICT": ""}) def test_DatasetDiff_summary(tmp_path): ds_a, ds_b = _create_datasets(tmp_path) @@ -43,6 +47,7 @@ def test_DatasetDiff_summary(tmp_path): ] +@patch.dict(os.environ, {"OWID_STRICT": ""}) def test_new_data(tmp_path): ds_a, ds_b = _create_datasets(tmp_path) diff --git a/tests/test_etl.py b/tests/test_etl.py index 006ded13886..7726e42307f 100644 --- a/tests/test_etl.py +++ b/tests/test_etl.py @@ -5,6 +5,7 @@ """ Check the integrity of the DAG. """ + from pathlib import Path from typing import List, Union from unittest.mock import patch diff --git a/tests/test_grapher_helpers.py b/tests/test_grapher_helpers.py index b54ba3d2f6d..b68fb92f1e2 100644 --- a/tests/test_grapher_helpers.py +++ b/tests/test_grapher_helpers.py @@ -222,12 +222,16 @@ def test_expand_jinja(): presentation=VariablePresentationMeta( title_variant="Variant << foo >>", ), + display={ + "isProjection": "<% if foo == 'bar' %>true<% else %>false<% endif %>", + }, ) out = gh._expand_jinja(m, dim_dict={"foo": "bar"}) assert out.to_dict() == { "title": "Title bar", "description_key": ["This is bar"], "presentation": {"title_variant": "Variant bar"}, + "display": {"isProjection": True}, } diff --git a/tests/test_steps.py b/tests/test_steps.py index ff266f1917d..5693fcd05fd 100644 --- a/tests/test_steps.py +++ b/tests/test_steps.py @@ -15,6 +15,7 @@ from unittest.mock import patch import pandas as pd +import requests from owid.catalog import Dataset from etl import paths @@ -162,7 +163,11 @@ def test_select_dirty_steps(): def test_get_etag(): - etag = get_etag("https://raw.githubusercontent.com/owid/owid-grapher/master/README.md") + try: + etag = get_etag("https://raw.githubusercontent.com/owid/owid-grapher/master/README.md") + # ignore SSL errors + except requests.exceptions.SSLError: + return assert etag diff --git a/uv.lock b/uv.lock index 37dd673c98a..ced25ca0fd1 100644 --- a/uv.lock +++ b/uv.lock @@ -373,6 +373,86 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ed/22/1b0892a70916439eefbd457c1d881a5a8c2ae17c671ddea05a2439515fd4/botocore_stubs-1.35.18-py3-none-any.whl", hash = "sha256:9f8cd580b2c9781f21d43bc5b3bf421a502270c81fd5ff235e655e92e91dc248", size = 60133 }, ] +[[package]] +name = "brotli" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/c2/f9e977608bdf958650638c3f1e28f85a1b075f075ebbe77db8555463787b/Brotli-1.1.0.tar.gz", hash = "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724", size = 7372270 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/3a/dbf4fb970c1019a57b5e492e1e0eae745d32e59ba4d6161ab5422b08eefe/Brotli-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e1140c64812cb9b06c922e77f1c26a75ec5e3f0fb2bf92cc8c58720dec276752", size = 873045 }, + { url = "https://files.pythonhosted.org/packages/dd/11/afc14026ea7f44bd6eb9316d800d439d092c8d508752055ce8d03086079a/Brotli-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c8fd5270e906eef71d4a8d19b7c6a43760c6abcfcc10c9101d14eb2357418de9", size = 446218 }, + { url = "https://files.pythonhosted.org/packages/36/83/7545a6e7729db43cb36c4287ae388d6885c85a86dd251768a47015dfde32/Brotli-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ae56aca0402a0f9a3431cddda62ad71666ca9d4dc3a10a142b9dce2e3c0cda3", size = 2903872 }, + { url = "https://files.pythonhosted.org/packages/32/23/35331c4d9391fcc0f29fd9bec2c76e4b4eeab769afbc4b11dd2e1098fb13/Brotli-1.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:43ce1b9935bfa1ede40028054d7f48b5469cd02733a365eec8a329ffd342915d", size = 2941254 }, + { url = "https://files.pythonhosted.org/packages/3b/24/1671acb450c902edb64bd765d73603797c6c7280a9ada85a195f6b78c6e5/Brotli-1.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:7c4855522edb2e6ae7fdb58e07c3ba9111e7621a8956f481c68d5d979c93032e", size = 2857293 }, + { url = "https://files.pythonhosted.org/packages/d5/00/40f760cc27007912b327fe15bf6bfd8eaecbe451687f72a8abc587d503b3/Brotli-1.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:38025d9f30cf4634f8309c6874ef871b841eb3c347e90b0851f63d1ded5212da", size = 3002385 }, + { url = "https://files.pythonhosted.org/packages/b8/cb/8aaa83f7a4caa131757668c0fb0c4b6384b09ffa77f2fba9570d87ab587d/Brotli-1.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e6a904cb26bfefc2f0a6f240bdf5233be78cd2488900a2f846f3c3ac8489ab80", size = 2911104 }, + { url = "https://files.pythonhosted.org/packages/bc/c4/65456561d89d3c49f46b7fbeb8fe6e449f13bdc8ea7791832c5d476b2faf/Brotli-1.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a37b8f0391212d29b3a91a799c8e4a2855e0576911cdfb2515487e30e322253d", size = 2809981 }, + { url = "https://files.pythonhosted.org/packages/05/1b/cf49528437bae28abce5f6e059f0d0be6fecdcc1d3e33e7c54b3ca498425/Brotli-1.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e84799f09591700a4154154cab9787452925578841a94321d5ee8fb9a9a328f0", size = 2935297 }, + { url = "https://files.pythonhosted.org/packages/81/ff/190d4af610680bf0c5a09eb5d1eac6e99c7c8e216440f9c7cfd42b7adab5/Brotli-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f66b5337fa213f1da0d9000bc8dc0cb5b896b726eefd9c6046f699b169c41b9e", size = 2930735 }, + { url = "https://files.pythonhosted.org/packages/80/7d/f1abbc0c98f6e09abd3cad63ec34af17abc4c44f308a7a539010f79aae7a/Brotli-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5dab0844f2cf82be357a0eb11a9087f70c5430b2c241493fc122bb6f2bb0917c", size = 2933107 }, + { url = "https://files.pythonhosted.org/packages/34/ce/5a5020ba48f2b5a4ad1c0522d095ad5847a0be508e7d7569c8630ce25062/Brotli-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e4fe605b917c70283db7dfe5ada75e04561479075761a0b3866c081d035b01c1", size = 2845400 }, + { url = "https://files.pythonhosted.org/packages/44/89/fa2c4355ab1eecf3994e5a0a7f5492c6ff81dfcb5f9ba7859bd534bb5c1a/Brotli-1.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1e9a65b5736232e7a7f91ff3d02277f11d339bf34099a56cdab6a8b3410a02b2", size = 3031985 }, + { url = "https://files.pythonhosted.org/packages/af/a4/79196b4a1674143d19dca400866b1a4d1a089040df7b93b88ebae81f3447/Brotli-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:58d4b711689366d4a03ac7957ab8c28890415e267f9b6589969e74b6e42225ec", size = 2927099 }, + { url = "https://files.pythonhosted.org/packages/e9/54/1c0278556a097f9651e657b873ab08f01b9a9ae4cac128ceb66427d7cd20/Brotli-1.1.0-cp310-cp310-win32.whl", hash = "sha256:be36e3d172dc816333f33520154d708a2657ea63762ec16b62ece02ab5e4daf2", size = 333172 }, + { url = "https://files.pythonhosted.org/packages/f7/65/b785722e941193fd8b571afd9edbec2a9b838ddec4375d8af33a50b8dab9/Brotli-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:0c6244521dda65ea562d5a69b9a26120769b7a9fb3db2fe9545935ed6735b128", size = 357255 }, + { url = "https://files.pythonhosted.org/packages/96/12/ad41e7fadd5db55459c4c401842b47f7fee51068f86dd2894dd0dcfc2d2a/Brotli-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a3daabb76a78f829cafc365531c972016e4aa8d5b4bf60660ad8ecee19df7ccc", size = 873068 }, + { url = "https://files.pythonhosted.org/packages/95/4e/5afab7b2b4b61a84e9c75b17814198ce515343a44e2ed4488fac314cd0a9/Brotli-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c8146669223164fc87a7e3de9f81e9423c67a79d6b3447994dfb9c95da16e2d6", size = 446244 }, + { url = "https://files.pythonhosted.org/packages/9d/e6/f305eb61fb9a8580c525478a4a34c5ae1a9bcb12c3aee619114940bc513d/Brotli-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30924eb4c57903d5a7526b08ef4a584acc22ab1ffa085faceb521521d2de32dd", size = 2906500 }, + { url = "https://files.pythonhosted.org/packages/3e/4f/af6846cfbc1550a3024e5d3775ede1e00474c40882c7bf5b37a43ca35e91/Brotli-1.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ceb64bbc6eac5a140ca649003756940f8d6a7c444a68af170b3187623b43bebf", size = 2943950 }, + { url = "https://files.pythonhosted.org/packages/b3/e7/ca2993c7682d8629b62630ebf0d1f3bb3d579e667ce8e7ca03a0a0576a2d/Brotli-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a469274ad18dc0e4d316eefa616d1d0c2ff9da369af19fa6f3daa4f09671fd61", size = 2918527 }, + { url = "https://files.pythonhosted.org/packages/b3/96/da98e7bedc4c51104d29cc61e5f449a502dd3dbc211944546a4cc65500d3/Brotli-1.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:524f35912131cc2cabb00edfd8d573b07f2d9f21fa824bd3fb19725a9cf06327", size = 2845489 }, + { url = "https://files.pythonhosted.org/packages/e8/ef/ccbc16947d6ce943a7f57e1a40596c75859eeb6d279c6994eddd69615265/Brotli-1.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5b3cc074004d968722f51e550b41a27be656ec48f8afaeeb45ebf65b561481dd", size = 2914080 }, + { url = "https://files.pythonhosted.org/packages/80/d6/0bd38d758d1afa62a5524172f0b18626bb2392d717ff94806f741fcd5ee9/Brotli-1.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:19c116e796420b0cee3da1ccec3b764ed2952ccfcc298b55a10e5610ad7885f9", size = 2813051 }, + { url = "https://files.pythonhosted.org/packages/14/56/48859dd5d129d7519e001f06dcfbb6e2cf6db92b2702c0c2ce7d97e086c1/Brotli-1.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:510b5b1bfbe20e1a7b3baf5fed9e9451873559a976c1a78eebaa3b86c57b4265", size = 2938172 }, + { url = "https://files.pythonhosted.org/packages/3d/77/a236d5f8cd9e9f4348da5acc75ab032ab1ab2c03cc8f430d24eea2672888/Brotli-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a1fd8a29719ccce974d523580987b7f8229aeace506952fa9ce1d53a033873c8", size = 2933023 }, + { url = "https://files.pythonhosted.org/packages/f1/87/3b283efc0f5cb35f7f84c0c240b1e1a1003a5e47141a4881bf87c86d0ce2/Brotli-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c247dd99d39e0338a604f8c2b3bc7061d5c2e9e2ac7ba9cc1be5a69cb6cd832f", size = 2935871 }, + { url = "https://files.pythonhosted.org/packages/f3/eb/2be4cc3e2141dc1a43ad4ca1875a72088229de38c68e842746b342667b2a/Brotli-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1b2c248cd517c222d89e74669a4adfa5577e06ab68771a529060cf5a156e9757", size = 2847784 }, + { url = "https://files.pythonhosted.org/packages/66/13/b58ddebfd35edde572ccefe6890cf7c493f0c319aad2a5badee134b4d8ec/Brotli-1.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:2a24c50840d89ded6c9a8fdc7b6ed3692ed4e86f1c4a4a938e1e92def92933e0", size = 3034905 }, + { url = "https://files.pythonhosted.org/packages/84/9c/bc96b6c7db824998a49ed3b38e441a2cae9234da6fa11f6ed17e8cf4f147/Brotli-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f31859074d57b4639318523d6ffdca586ace54271a73ad23ad021acd807eb14b", size = 2929467 }, + { url = "https://files.pythonhosted.org/packages/e7/71/8f161dee223c7ff7fea9d44893fba953ce97cf2c3c33f78ba260a91bcff5/Brotli-1.1.0-cp311-cp311-win32.whl", hash = "sha256:39da8adedf6942d76dc3e46653e52df937a3c4d6d18fdc94a7c29d263b1f5b50", size = 333169 }, + { url = "https://files.pythonhosted.org/packages/02/8a/fece0ee1057643cb2a5bbf59682de13f1725f8482b2c057d4e799d7ade75/Brotli-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:aac0411d20e345dc0920bdec5548e438e999ff68d77564d5e9463a7ca9d3e7b1", size = 357253 }, + { url = "https://files.pythonhosted.org/packages/5c/d0/5373ae13b93fe00095a58efcbce837fd470ca39f703a235d2a999baadfbc/Brotli-1.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:32d95b80260d79926f5fab3c41701dbb818fde1c9da590e77e571eefd14abe28", size = 815693 }, + { url = "https://files.pythonhosted.org/packages/8e/48/f6e1cdf86751300c288c1459724bfa6917a80e30dbfc326f92cea5d3683a/Brotli-1.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b760c65308ff1e462f65d69c12e4ae085cff3b332d894637f6273a12a482d09f", size = 422489 }, + { url = "https://files.pythonhosted.org/packages/06/88/564958cedce636d0f1bed313381dfc4b4e3d3f6015a63dae6146e1b8c65c/Brotli-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409", size = 873081 }, + { url = "https://files.pythonhosted.org/packages/58/79/b7026a8bb65da9a6bb7d14329fd2bd48d2b7f86d7329d5cc8ddc6a90526f/Brotli-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2", size = 446244 }, + { url = "https://files.pythonhosted.org/packages/e5/18/c18c32ecea41b6c0004e15606e274006366fe19436b6adccc1ae7b2e50c2/Brotli-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451", size = 2906505 }, + { url = "https://files.pythonhosted.org/packages/08/c8/69ec0496b1ada7569b62d85893d928e865df29b90736558d6c98c2031208/Brotli-1.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7f4bf76817c14aa98cc6697ac02f3972cb8c3da93e9ef16b9c66573a68014f91", size = 2944152 }, + { url = "https://files.pythonhosted.org/packages/ab/fb/0517cea182219d6768113a38167ef6d4eb157a033178cc938033a552ed6d/Brotli-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0c5516f0aed654134a2fc936325cc2e642f8a0e096d075209672eb321cff408", size = 2919252 }, + { url = "https://files.pythonhosted.org/packages/c7/53/73a3431662e33ae61a5c80b1b9d2d18f58dfa910ae8dd696e57d39f1a2f5/Brotli-1.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c3020404e0b5eefd7c9485ccf8393cfb75ec38ce75586e046573c9dc29967a0", size = 2845955 }, + { url = "https://files.pythonhosted.org/packages/55/ac/bd280708d9c5ebdbf9de01459e625a3e3803cce0784f47d633562cf40e83/Brotli-1.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4ed11165dd45ce798d99a136808a794a748d5dc38511303239d4e2363c0695dc", size = 2914304 }, + { url = "https://files.pythonhosted.org/packages/76/58/5c391b41ecfc4527d2cc3350719b02e87cb424ef8ba2023fb662f9bf743c/Brotli-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180", size = 2814452 }, + { url = "https://files.pythonhosted.org/packages/c7/4e/91b8256dfe99c407f174924b65a01f5305e303f486cc7a2e8a5d43c8bec3/Brotli-1.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248", size = 2938751 }, + { url = "https://files.pythonhosted.org/packages/5a/a6/e2a39a5d3b412938362bbbeba5af904092bf3f95b867b4a3eb856104074e/Brotli-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966", size = 2933757 }, + { url = "https://files.pythonhosted.org/packages/13/f0/358354786280a509482e0e77c1a5459e439766597d280f28cb097642fc26/Brotli-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87a3044c3a35055527ac75e419dfa9f4f3667a1e887ee80360589eb8c90aabb9", size = 2936146 }, + { url = "https://files.pythonhosted.org/packages/80/f7/daf538c1060d3a88266b80ecc1d1c98b79553b3f117a485653f17070ea2a/Brotli-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c5529b34c1c9d937168297f2c1fde7ebe9ebdd5e121297ff9c043bdb2ae3d6fb", size = 2848055 }, + { url = "https://files.pythonhosted.org/packages/ad/cf/0eaa0585c4077d3c2d1edf322d8e97aabf317941d3a72d7b3ad8bce004b0/Brotli-1.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ca63e1890ede90b2e4454f9a65135a4d387a4585ff8282bb72964fab893f2111", size = 3035102 }, + { url = "https://files.pythonhosted.org/packages/d8/63/1c1585b2aa554fe6dbce30f0c18bdbc877fa9a1bf5ff17677d9cca0ac122/Brotli-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e79e6520141d792237c70bcd7a3b122d00f2613769ae0cb61c52e89fd3443839", size = 2930029 }, + { url = "https://files.pythonhosted.org/packages/5f/3b/4e3fd1893eb3bbfef8e5a80d4508bec17a57bb92d586c85c12d28666bb13/Brotli-1.1.0-cp312-cp312-win32.whl", hash = "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0", size = 333276 }, + { url = "https://files.pythonhosted.org/packages/3d/d5/942051b45a9e883b5b6e98c041698b1eb2012d25e5948c58d6bf85b1bb43/Brotli-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951", size = 357255 }, +] + +[[package]] +name = "brotlicffi" +version = "1.1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/9d/70caa61192f570fcf0352766331b735afa931b4c6bc9a348a0925cc13288/brotlicffi-1.1.0.0.tar.gz", hash = "sha256:b77827a689905143f87915310b93b273ab17888fd43ef350d4832c4a71083c13", size = 465192 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/11/7b96009d3dcc2c931e828ce1e157f03824a69fb728d06bfd7b2fc6f93718/brotlicffi-1.1.0.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9b7ae6bd1a3f0df532b6d67ff674099a96d22bc0948955cb338488c31bfb8851", size = 453786 }, + { url = "https://files.pythonhosted.org/packages/d6/e6/a8f46f4a4ee7856fbd6ac0c6fb0dc65ed181ba46cd77875b8d9bbe494d9e/brotlicffi-1.1.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19ffc919fa4fc6ace69286e0a23b3789b4219058313cf9b45625016bf7ff996b", size = 2911165 }, + { url = "https://files.pythonhosted.org/packages/be/20/201559dff14e83ba345a5ec03335607e47467b6633c210607e693aefac40/brotlicffi-1.1.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9feb210d932ffe7798ee62e6145d3a757eb6233aa9a4e7db78dd3690d7755814", size = 2927895 }, + { url = "https://files.pythonhosted.org/packages/cd/15/695b1409264143be3c933f708a3f81d53c4a1e1ebbc06f46331decbf6563/brotlicffi-1.1.0.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84763dbdef5dd5c24b75597a77e1b30c66604725707565188ba54bab4f114820", size = 2851834 }, + { url = "https://files.pythonhosted.org/packages/b4/40/b961a702463b6005baf952794c2e9e0099bde657d0d7e007f923883b907f/brotlicffi-1.1.0.0-cp37-abi3-win32.whl", hash = "sha256:1b12b50e07c3911e1efa3a8971543e7648100713d4e0971b13631cce22c587eb", size = 341731 }, + { url = "https://files.pythonhosted.org/packages/1c/fa/5408a03c041114ceab628ce21766a4ea882aa6f6f0a800e04ee3a30ec6b9/brotlicffi-1.1.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:994a4f0681bb6c6c3b0925530a1926b7a189d878e6e5e38fae8efa47c5d9c613", size = 366783 }, + { url = "https://files.pythonhosted.org/packages/e5/3b/bd4f3d2bcf2306ae66b0346f5b42af1962480b200096ffc7abc3bd130eca/brotlicffi-1.1.0.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:2e4aeb0bd2540cb91b069dbdd54d458da8c4334ceaf2d25df2f4af576d6766ca", size = 397397 }, + { url = "https://files.pythonhosted.org/packages/54/10/1fd57864449360852c535c2381ee7120ba8f390aa3869df967c44ca7eba1/brotlicffi-1.1.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b7b0033b0d37bb33009fb2fef73310e432e76f688af76c156b3594389d81391", size = 379698 }, + { url = "https://files.pythonhosted.org/packages/e5/95/15aa422aa6450e6556e54a5fd1650ff59f470aed77ac739aa90ab63dc611/brotlicffi-1.1.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54a07bb2374a1eba8ebb52b6fafffa2afd3c4df85ddd38fcc0511f2bb387c2a8", size = 378635 }, + { url = "https://files.pythonhosted.org/packages/6c/a7/f254e13b2cb43337d6d99a4ec10394c134e41bfda8a2eff15b75627f4a3d/brotlicffi-1.1.0.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7901a7dc4b88f1c1475de59ae9be59799db1007b7d059817948d8e4f12e24e35", size = 385719 }, + { url = "https://files.pythonhosted.org/packages/72/a9/0971251c4427c14b2a827dba3d910d4d3330dabf23d4278bf6d06a978847/brotlicffi-1.1.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ce01c7316aebc7fce59da734286148b1d1b9455f89cf2c8a4dfce7d41db55c2d", size = 361760 }, +] + [[package]] name = "bugsnag" version = "4.7.1" @@ -837,6 +917,7 @@ dependencies = [ { name = "frictionless", extra = ["pandas"] }, { name = "fsspec" }, { name = "geopandas" }, + { name = "geopy" }, { name = "gitpython" }, { name = "h5netcdf" }, { name = "html2text" }, @@ -851,11 +932,13 @@ dependencies = [ { name = "pandas" }, { name = "papermill" }, { name = "pdfplumber" }, + { name = "py7zr" }, { name = "pydantic" }, { name = "pygithub" }, { name = "pyhumps" }, { name = "pymysql" }, { name = "pypdf2" }, + { name = "pyreadr" }, { name = "python-docx" }, { name = "python-dotenv" }, { name = "pyyaml" }, @@ -951,10 +1034,11 @@ requires-dist = [ { name = "earthengine-api", specifier = ">=0.1.411" }, { name = "fastapi", marker = "extra == 'api'", specifier = ">=0.109.0" }, { name = "fasteners", specifier = ">=0.19" }, - { name = "frictionless", extras = ["pandas"], specifier = ">=4.40.8,<5.0.0" }, - { name = "fsspec", specifier = "==2022.11.0" }, + { name = "frictionless", extras = ["pandas"], specifier = ">=5.0.3" }, + { name = "fsspec", specifier = ">=2022.11.0" }, { name = "geographiclib", marker = "extra == 'wizard'", specifier = ">=2.0" }, { name = "geopandas", specifier = ">=0.14.1" }, + { name = "geopy", specifier = ">=2.4.1" }, { name = "gitpython", specifier = ">=3.1.30" }, { name = "h5netcdf", specifier = ">=1.3.0" }, { name = "html2text", specifier = ">=2020.1.16" }, @@ -972,12 +1056,14 @@ requires-dist = [ { name = "papermill", specifier = ">=2.3.3" }, { name = "pdfplumber", specifier = ">=0.9.0" }, { name = "plotly", marker = "extra == 'wizard'", specifier = ">=5.23.0" }, + { name = "py7zr", specifier = ">=0.22.0" }, { name = "pydantic", specifier = ">=1.9.0" }, { name = "pygithub", specifier = ">=2.3.0" }, { name = "pyhumps", specifier = ">=3.8.0" }, { name = "pymysql", specifier = ">=1.1.1" }, { name = "pypdf2", specifier = ">=2.11.1" }, { name = "pyproj", marker = "extra == 'wizard'", specifier = ">=3.6.1" }, + { name = "pyreadr", specifier = ">=0.5.2" }, { name = "python-docx", specifier = ">=1.1.2" }, { name = "python-dotenv", specifier = ">=0.19.0" }, { name = "pyyaml", specifier = ">=6.0.1" }, @@ -1046,7 +1132,7 @@ dev = [ { name = "pyright", specifier = "==1.1.373" }, { name = "pytest", specifier = ">=8.3.2" }, { name = "pyyaml", specifier = ">=6.0.2" }, - { name = "ruff", specifier = "==0.1.6" }, + { name = "ruff", specifier = ">=0.8.2" }, { name = "sqlacodegen", git = "https://github.com/agronholm/sqlacodegen.git" }, { name = "types-pyyaml", specifier = ">=6.0.12.20240808" }, { name = "watchdog", specifier = ">=4.0.1" }, @@ -1207,15 +1293,18 @@ wheels = [ [[package]] name = "frictionless" -version = "4.40.11" +version = "5.18.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "attrs" }, { name = "chardet" }, + { name = "humanize" }, { name = "isodate" }, { name = "jinja2" }, { name = "jsonschema" }, { name = "marko" }, { name = "petl" }, + { name = "pydantic" }, { name = "python-dateutil" }, { name = "python-slugify" }, { name = "pyyaml" }, @@ -1225,16 +1314,18 @@ dependencies = [ { name = "stringcase" }, { name = "tabulate" }, { name = "typer" }, + { name = "typing-extensions" }, { name = "validators" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/56/90/6e4126b50d4edeffcd5f39657b4648fb6f3d17d43088ab09d62f49e1cad0/frictionless-4.40.11.tar.gz", hash = "sha256:e7d83d82cd3273820c74ac715e8d78285697f1eceda49a2417a72f839420d42e", size = 258035 } +sdist = { url = "https://files.pythonhosted.org/packages/26/b4/ded94e51965f95100893adcf78ef9307553414a0bb56217adf68450bd7e7/frictionless-5.18.0.tar.gz", hash = "sha256:4b21a10d3ac67e46a4a58a1e8a8a27c6882af4d1608eadfb6ccbfde0b5eef6b9", size = 74371639 } wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/cb/13b97bcf9c2ed6a4dc3b7d6fe99f7d7a1f395a2847ca3d951afbf82d6787/frictionless-4.40.11-py2.py3-none-any.whl", hash = "sha256:5b2bbb3779d5e2ecfe99add2458a7b2bcb61eae6173696ea57ef0b28c085d976", size = 419905 }, + { url = "https://files.pythonhosted.org/packages/fb/e5/c7ff55b81286f24ddfaff45c9d46614c3e40c72a8ebd036c2cc18d902243/frictionless-5.18.0-py3-none-any.whl", hash = "sha256:a82433b81cfcfae21328aad6b93854feb86d5d054b22ac147672eb9c254b6a3d", size = 535385 }, ] [package.optional-dependencies] pandas = [ { name = "pandas" }, + { name = "pyarrow" }, ] [[package]] @@ -1341,6 +1432,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3e/b0/69fa7a0f55122847506a42fea6988d03b34136938082f142151bc9d9f7e7/geopandas-0.14.4-py3-none-any.whl", hash = "sha256:3bb6473cb59d51e1a7fe2dbc24a1a063fb0ebdeddf3ce08ddbf8c7ddc99689aa", size = 1109913 }, ] +[[package]] +name = "geopy" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "geographiclib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/fd/ef6d53875ceab72c1fad22dbed5ec1ad04eb378c2251a6a8024bad890c3b/geopy-2.4.1.tar.gz", hash = "sha256:50283d8e7ad07d89be5cb027338c6365a32044df3ae2556ad3f52f4840b3d0d1", size = 117625 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/15/cf2a69ade4b194aa524ac75112d5caac37414b20a3a03e6865dfe0bd1539/geopy-2.4.1-py3-none-any.whl", hash = "sha256:ae8b4bc5c1131820f4d75fce9d4aaaca0c85189b3aa5d64c3dcaf5e3b7b882a7", size = 125437 }, +] + [[package]] name = "ghp-import" version = "2.1.0" @@ -1744,6 +1847,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/f3/3fc97336a0e90516901befd4f500f08d691034d387406fdbde85bea827cc/huggingface_hub-0.17.3-py3-none-any.whl", hash = "sha256:545eb3665f6ac587add946e73984148f2ea5c7877eac2e845549730570c1933a", size = 295010 }, ] +[[package]] +name = "humanize" +version = "4.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/40/64a912b9330786df25e58127194d4a5a7441f818b400b155e748a270f924/humanize-4.11.0.tar.gz", hash = "sha256:e66f36020a2d5a974c504bd2555cf770621dbdbb6d82f94a6857c0b1ea2608be", size = 80374 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/75/4bc3e242ad13f2e6c12e0b0401ab2c5e5c6f0d7da37ec69bc808e24e0ccb/humanize-4.11.0-py3-none-any.whl", hash = "sha256:b53caaec8532bcb2fff70c8826f904c35943f8cecaca29d272d9df38092736c0", size = 128055 }, +] + [[package]] name = "hydra-core" version = "1.3.2" @@ -1796,6 +1908,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a9/1c/1b9c72bf839def47626436ea5ebaf643404f7850482c5fafd71a3deeaa94/imageio_ffmpeg-0.5.1-py3-none-win_amd64.whl", hash = "sha256:1521e79e253bedbdd36a547e0cbd94a025ba0b558e17f08fea687d805a0e4698", size = 22619891 }, ] +[[package]] +name = "inflate64" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/99/18f9940d4a3f2cabc4396a587ddf1bd93236bdb372d9e78e2b0365e40990/inflate64-1.0.0.tar.gz", hash = "sha256:3278827b803cf006a1df251f3e13374c7d26db779e5a33329cc11789b804bc2d", size = 895853 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/cf/06af80e81dd4bbb7e883291cf1726035d526f066a37c4ed4d4cd88a7a49d/inflate64-1.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a90c0bdf4a7ecddd8a64cc977181810036e35807f56b0bcacee9abb0fcfd18dc", size = 59418 }, + { url = "https://files.pythonhosted.org/packages/c9/4b/6f18918220b1a8e935121cece1dc917e62fa593fc637a621470f9b9a601a/inflate64-1.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:57fe7c14aebf1c5a74fc3b70d355be1280a011521a76aa3895486e62454f4242", size = 36231 }, + { url = "https://files.pythonhosted.org/packages/aa/f4/f4b5dbd78dd5af66b6ca32778ebaa9c14d67b68ea84e96592ccf40786a41/inflate64-1.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d90730165f471d61a1a694a5e354f3ffa938227e8dcecb62d5d728e8069cee94", size = 35738 }, + { url = "https://files.pythonhosted.org/packages/10/23/26289a700550767cf5eb7550f78ad826529706287393f224bbaee3c1b1e2/inflate64-1.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:543f400201f5c101141af3c79c82059e1aa6ef4f1584a7f1fa035fb2e465097f", size = 92855 }, + { url = "https://files.pythonhosted.org/packages/b8/f4/e387a50f5027194eac4f9712d57b97e3e1a012402eaae98bcf1ebe8a97d1/inflate64-1.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ceca14f7ec19fb44b047f56c50efb7521b389d222bba2b0a10286a0caeb03fa", size = 93141 }, + { url = "https://files.pythonhosted.org/packages/33/c8/e516aecd9ed0dc75d8df041ed4ef80f2e2be39d0e516c7269b7f274e760a/inflate64-1.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b559937a42f0c175b4d2dfc7eb53b97bdc87efa9add15ed5549c6abc1e89d02f", size = 95262 }, + { url = "https://files.pythonhosted.org/packages/0b/aa/ed3ab5f8c13afc432fb382edf97cede7a6f9be73ecf98bfe64b686c8d223/inflate64-1.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5ff8bd2a562343fcbc4eea26fdc368904a3b5f6bb8262344274d3d74a1de15bb", size = 95912 }, + { url = "https://files.pythonhosted.org/packages/e0/64/5637c4f67ed15518c0765b85b528ed79536caaf8ba167a9f7173e334d4a8/inflate64-1.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:0fe481f31695d35a433c3044ac8fd5d9f5069aaad03a0c04b570eb258ce655aa", size = 35166 }, + { url = "https://files.pythonhosted.org/packages/af/92/701b3c76b1cf244026c3e78dff8487955cf6960c1d9f350e2820a0d1a5d9/inflate64-1.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a45f6979ad5874d4d4898c2fc770b136e61b96b850118fdaec5a5af1b9123a", size = 59450 }, + { url = "https://files.pythonhosted.org/packages/bb/1d/af0253fafc27cadd29e3b111ebb3011b8c913a3554b403c90c7595f5933e/inflate64-1.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:022ca1cc928e7365a05f7371ff06af143c6c667144965e2cf9a9236a2ae1c291", size = 36267 }, + { url = "https://files.pythonhosted.org/packages/b6/22/7949030be11f4754bd6ed7067e9bebdf614013b89ccd4638330a85821b51/inflate64-1.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:46792ecf3565d64fd2c519b0a780c03a57e195613c9954ef94e739a057b3fd06", size = 35740 }, + { url = "https://files.pythonhosted.org/packages/e4/87/c6ce0093a345c04811f6171a367665dec17dcc4617ca150dd37e9ae7bd33/inflate64-1.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a70ea2e456c15f7aa7c74b8ab8f20b4f8940ec657604c9f0a9de3342f280fff", size = 95896 }, + { url = "https://files.pythonhosted.org/packages/62/d6/fe113b12773cad2c093d381c2b1629f9cfa240c9ad86a7f9f9079e7a51b5/inflate64-1.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e243ea9bd36a035059f2365bd6d156ff59717fbafb0255cb0c75bf151bf6904", size = 96007 }, + { url = "https://files.pythonhosted.org/packages/f0/a6/9165bee4b7fc5af949fec12a2cea7ad73bf9ee97dfb96a0276274c48e709/inflate64-1.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4dc392dec1cd11cacda3d2637214ca45e38202e8a4f31d4a4e566d6e90625fc4", size = 98297 }, + { url = "https://files.pythonhosted.org/packages/ee/72/0aeb360101eeed32696fc6c623bc1780fac895a9fc2e93b582cb1e22ca54/inflate64-1.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8b402a50eda7ee75f342fc346d33a41bca58edc222a4b17f9be0db1daed459fa", size = 98858 }, + { url = "https://files.pythonhosted.org/packages/94/4a/8301ad59b57d9de504b0fdce22bf980dfb231753e6d7aed12af938f7f9fd/inflate64-1.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:f5924499dc8800928c0ee4580fa8eb4ffa880b2cce4431537d0390e503a9c9ee", size = 35167 }, + { url = "https://files.pythonhosted.org/packages/18/82/47021b8919c1dc276d0502296f15ffac1cd648b94b35cadb14cb812b6199/inflate64-1.0.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0c644bf7208e20825ca3bbb5fb1f7f495cfcb49eb01a5f67338796d44a42f2bf", size = 59509 }, + { url = "https://files.pythonhosted.org/packages/e0/c9/00701be8e48dc9c9b9488001d9c66d6cb6f6bb0c48af9abf33a69726d130/inflate64-1.0.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9964a4eaf26a9d36f82a1d9b12c28e35800dd3d99eb340453ed12ac90c2976a8", size = 36305 }, + { url = "https://files.pythonhosted.org/packages/25/c0/11dea5e298b2e7d61f0fbd1005553e8796e35536751980b676547fcc57ef/inflate64-1.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2cccded63865640d03253897be7232b2bbac295fe43914c61f86a57aa23bb61d", size = 35756 }, + { url = "https://files.pythonhosted.org/packages/86/ba/4debdaaafdc21853621caf463a498a754ee4352893454c596dbd65294e9f/inflate64-1.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d491f104fb3701926ebd82b8c9250dfba0ddcab584504e26f1e4adb26730378d", size = 96127 }, + { url = "https://files.pythonhosted.org/packages/89/81/8f559c199ec13d0b70d0dc46811490b2976873c96c564941583777e9b343/inflate64-1.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ebad4a6cd2a2c1d81be0b09d4006479f3b258803c49a9224ef8ca0b649072fa", size = 96903 }, + { url = "https://files.pythonhosted.org/packages/46/41/39ac4c7e17d0690578b716a0ff34e00600616994795b0645fd61fc600c0f/inflate64-1.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:6823b2c0cff3a8159140f3b17ec64fb8ec0e663b45a6593618ecdde8aeecb5b2", size = 98855 }, + { url = "https://files.pythonhosted.org/packages/44/dd/be5d69492c180f94a6af8a15564ce365bdcb84bd1a6fb32949d6913959aa/inflate64-1.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:228d504239d27958e71fc77e3119a6ac4528127df38468a0c95a5bd3927204b8", size = 99884 }, + { url = "https://files.pythonhosted.org/packages/8c/0d/a5266bd4f2cdb7fad1eae3ffe4dcc16f9769323660a0a6cfbe9cc1d2cf03/inflate64-1.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:ae2572e06bcfe15e3bbf77d4e4a6d6c55e2a70d6abceaaf60c5c3653ddb96dfd", size = 35334 }, + { url = "https://files.pythonhosted.org/packages/53/91/43238dd8a7e5bab71abae872c09931db4b31aebf672afccb305f79aacb3e/inflate64-1.0.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f79542478e49e471e8b23556700e6f688a40dc93e9a746f77a546c13251b59b1", size = 34648 }, + { url = "https://files.pythonhosted.org/packages/ef/6f/ce090934a80c1fd0b5b07c125ed6eb2845f11a78af344d69c0f051dcab97/inflate64-1.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a270be6b10cde01258c0097a663a307c62d12c78eb8f62f8e29f205335942c9", size = 36473 }, + { url = "https://files.pythonhosted.org/packages/b4/fe/2cd4bf78696213b807860002c182dd1751ba52c1559143b1b8daa7904733/inflate64-1.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1616a87ff04f583e9558cc247ec0b72a30d540ee0c17cc77823be175c0ec92f0", size = 36478 }, + { url = "https://files.pythonhosted.org/packages/43/dd/e62444c0ef7d1228b622e6d3dacf9ea237d8807a78619a83832a3b4a5adf/inflate64-1.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:137ca6b315f0157a786c3a755a09395ca69aed8bcf42ad3437cb349f5ebc86d2", size = 35630 }, +] + [[package]] name = "inflect" version = "7.4.0" @@ -2903,6 +3051,15 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/cc/12/4e979f71d90ca5625647f93cd484c733a7e8ae4fd9f6d15369613d727301/multiurl-0.3.1.tar.gz", hash = "sha256:c7001437b59d56d4c310d725c3dcfff98c97c4b652893d88989853827465d442", size = 18161 } +[[package]] +name = "multivolumefile" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/50/f0/a7786212b5a4cb9ba05ae84a2bbd11d1d0279523aea0424b6d981d652a14/multivolumefile-0.2.3.tar.gz", hash = "sha256:a0648d0aafbc96e59198d5c17e9acad7eb531abea51035d08ce8060dcad709d6", size = 77984 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/31/ec5f46fd4c83185b806aa9c736e228cb780f13990a9cf4da0beb70025fcc/multivolumefile-0.2.3-py3-none-any.whl", hash = "sha256:237f4353b60af1703087cf7725755a1f6fcaeeea48421e1896940cd1c920d678", size = 17037 }, +] + [[package]] name = "mypy-boto3-s3" version = "1.35.16" @@ -3331,7 +3488,7 @@ dev = [ { name = "pyright", specifier = "==1.1.373" }, { name = "pytest", specifier = ">=6.2.5" }, { name = "pytest-cov", specifier = ">=2.12.1" }, - { name = "ruff", specifier = "==0.1.6" }, + { name = "ruff", specifier = "==0.8.2" }, { name = "watchdog", specifier = ">=2.1.5" }, ] @@ -3346,6 +3503,7 @@ dependencies = [ { name = "gdown" }, { name = "gsheets" }, { name = "pandas" }, + { name = "py7zr" }, { name = "pyarrow" }, { name = "pydrive2" }, { name = "structlog" }, @@ -3360,6 +3518,7 @@ requires-dist = [ { name = "gdown", specifier = ">=4.5.2" }, { name = "gsheets", specifier = ">=0.6.1" }, { name = "pandas", specifier = ">=2.2.1" }, + { name = "py7zr", specifier = ">=0.22.0" }, { name = "pyarrow", specifier = ">=10.0.1" }, { name = "pydrive2", specifier = ">=1.15.0" }, { name = "structlog", specifier = ">=21.5.0" }, @@ -3378,7 +3537,7 @@ dev = [ { name = "pyright", specifier = "==1.1.373" }, { name = "pytest", specifier = ">=6.2.5" }, { name = "pytest-cov", specifier = ">=2.12.1" }, - { name = "ruff", specifier = "==0.1.6" }, + { name = "ruff", specifier = "==0.8.2" }, { name = "sphinx", specifier = "==4.5.0" }, { name = "watchdog", specifier = ">=2.1.5" }, ] @@ -3405,7 +3564,7 @@ dev = [ { name = "ipdb", specifier = ">=0.13.13" }, { name = "pyright", specifier = "==1.1.373" }, { name = "pytest", specifier = ">=7.2.0" }, - { name = "ruff", specifier = "==0.1.6" }, + { name = "ruff", specifier = "==0.8.2" }, ] [[package]] @@ -3813,6 +3972,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842 }, ] +[[package]] +name = "py7zr" +version = "0.22.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "brotli", marker = "platform_python_implementation == 'CPython'" }, + { name = "brotlicffi", marker = "platform_python_implementation == 'PyPy'" }, + { name = "inflate64" }, + { name = "multivolumefile" }, + { name = "psutil", marker = "sys_platform != 'cygwin'" }, + { name = "pybcj" }, + { name = "pycryptodomex" }, + { name = "pyppmd" }, + { name = "pyzstd" }, + { name = "texttable" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/c3/0e05c711c16af0b9c47f3f77323303b338b9a871ba020d95d2b8dd6605ae/py7zr-0.22.0.tar.gz", hash = "sha256:c6c7aea5913535184003b73938490f9a4d8418598e533f9ca991d3b8e45a139e", size = 4992926 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/59/dd1750002c0f46099281116f8165247bc62dc85edad41cdd26e7b26de19d/py7zr-0.22.0-py3-none-any.whl", hash = "sha256:993b951b313500697d71113da2681386589b7b74f12e48ba13cc12beca79d078", size = 67906 }, +] + [[package]] name = "pyarrow" version = "17.0.0" @@ -3866,6 +4046,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/89/bc88a6711935ba795a679ea6ebee07e128050d6382eaa35a0a47c8032bdc/pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd", size = 181537 }, ] +[[package]] +name = "pybcj" +version = "1.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/d2/22e808b9d25ce3b43f5c8a9e22d873d403485ba55d84a4d6d5d044881762/pybcj-1.0.2.tar.gz", hash = "sha256:c7f5bef7f47723c53420e377bc64d2553843bee8bcac5f0ad076ab1524780018", size = 2111002 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/93/4735636b5905b7597068a2c7a10a8df0f668f28659207c274d64a4468b97/pybcj-1.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7bff28d97e47047d69a4ac6bf59adda738cf1d00adde8819117fdb65d966bdbc", size = 32556 }, + { url = "https://files.pythonhosted.org/packages/a6/37/443cd704397b6df54ff0822032e4815aca4e9badabc5ce1faac34235a40c/pybcj-1.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:198e0b4768b4025eb3309273d7e81dc53834b9a50092be6e0d9b3983cfd35c35", size = 23751 }, + { url = "https://files.pythonhosted.org/packages/9a/aa/5a19ed8661e979a4d3237a11706f9a16a474a2227fdd99ccb284be100a98/pybcj-1.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fa26415b4a118ea790de9d38f244312f2510a9bb5c65e560184d241a6f391a2d", size = 23980 }, + { url = "https://files.pythonhosted.org/packages/fe/5f/638ce03948905d267c8c0ccab81b8b4943a0324f63d8bdb0a0e2a85d4503/pybcj-1.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fabb2be57e4ca28ea36c13146cdf97d73abd27c51741923fc6ba1e8cd33e255c", size = 50155 }, + { url = "https://files.pythonhosted.org/packages/09/70/8b6a6cc2a5721f67f629bdc17875c0d603d57f360a19b099a7b4de19383d/pybcj-1.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75d6d613bae6f27678d5e44e89d61018779726aa6aa950c516d33a04b8af8c59", size = 49729 }, + { url = "https://files.pythonhosted.org/packages/89/06/2e41e34da0bb2adb3644cbf4366c344e5804a10f1153da7b3a23333f7db8/pybcj-1.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3ffae79ef8a1ea81ea2748ad7b7ad9b882aa88ddf65ce90f9e944df639eccc61", size = 54310 }, + { url = "https://files.pythonhosted.org/packages/b5/0f/de9e76c305d4dcd9d428a90ccac030f06c780bc30549fc449a944a6321bc/pybcj-1.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bdb4d8ff5cba3e0bd1adee7d20dbb2b4d80cb31ac04d6ea1cd06cfc02d2ecd0d", size = 53679 }, + { url = "https://files.pythonhosted.org/packages/1a/41/a807ff6b77ec8e49c749ed1d0db5649fbb1150c6fb5fb391115f4f1d743a/pybcj-1.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a29be917fbc99eca204b08407e0971e0205bfdad4b74ec915930675f352b669d", size = 24690 }, + { url = "https://files.pythonhosted.org/packages/27/0a/20bf70a7eb7c6b2668ff2af798254033c32a09d6c58ec9a87cd6aa843df5/pybcj-1.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a2562ebe5a0abec4da0229f8abb5e90ee97b178f19762eb925c1159be36828b3", size = 32581 }, + { url = "https://files.pythonhosted.org/packages/a9/b6/43977fe4296d2778c6dc67b596bb6a851eaea80f3dd4ff454e5fca8142c2/pybcj-1.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:af19bc61ded933001cd68f004ae2042bf1a78eb498a3c685ebd655fa1be90dbe", size = 23767 }, + { url = "https://files.pythonhosted.org/packages/89/c7/a61010f59406b8a45bb4865faa4b61d6b177dcfac04247fb56c7538d997d/pybcj-1.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f3f4a447800850aba7724a2274ea0a4800724520c1caf38f7d0dabf2f89a5e15", size = 23976 }, + { url = "https://files.pythonhosted.org/packages/10/7a/78848edbb6f12d9b86e375fc46135d9a204ededbf96682b05cb4b4fbd942/pybcj-1.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce1c8af7a4761d2b1b531864d84113948daa0c4245775c63bd9874cb955f4662", size = 51246 }, + { url = "https://files.pythonhosted.org/packages/9e/13/af86c86cdfb293e82dd0b6c4bbdf08645cd8993456ee3fb911c3eeed1b22/pybcj-1.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8007371f6f2b462f5aa05d5c2135d0a1bcf5b7bdd9bd15d86c730f588d10b7d3", size = 50754 }, + { url = "https://files.pythonhosted.org/packages/39/52/88600aa374b100612a1d82fca4b03eb4315e0084a05ee314ba1b771f7190/pybcj-1.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1079ca63ff8da5c936b76863690e0bd2489e8d4e0a3a340e032095dae805dd91", size = 55334 }, + { url = "https://files.pythonhosted.org/packages/56/67/3cf9747ef5b53e16a844217c6c9840be6289d05ec785500da2cc55cc25f2/pybcj-1.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e9a785eb26884429d9b9f6326e68c3638828c83bf6d42d2463c97ad5385caff2", size = 54714 }, + { url = "https://files.pythonhosted.org/packages/78/81/a71197903b503f54b85f4d352f909e701e9d26953577bd34d3fbe0520d5d/pybcj-1.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:9ea46e2d45469d13b7f25b08efcdb140220bab1ac5a850db0954591715b8caaa", size = 24693 }, + { url = "https://files.pythonhosted.org/packages/83/60/a3b43836895654aa93b5a8422adc3717359db98da9147abfabffef79f1e7/pybcj-1.0.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:21b5f2460629167340403d359289a173e0729ce8e84e3ce99462009d5d5e01a4", size = 32677 }, + { url = "https://files.pythonhosted.org/packages/50/b9/96c8d9577b0f5a701e4497408e6a331a08eb902aca8dfd4c5bb1eaab4779/pybcj-1.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:2940fb85730b9869254559c491cd83cf777e56c76a8a60df60e4be4f2a4248d7", size = 23813 }, + { url = "https://files.pythonhosted.org/packages/b7/1a/c80132feb084ec4098c0315a132799bddda8878113b5f956e21c4377f5f1/pybcj-1.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f40f3243139d675f43793a4e35c410c370f7b91ccae74e70c8b2f4877869f90e", size = 24019 }, + { url = "https://files.pythonhosted.org/packages/b1/94/62c3bf8a60b4787b46e21f43277d9cb8b6037c8ee183450f035a19a2bc4b/pybcj-1.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c2b3e60b65c7ac73e44335934e1e122da8d56db87840984601b3c5dc0ae4c19", size = 51927 }, + { url = "https://files.pythonhosted.org/packages/8b/9e/4ebd092251ef8d15408388be508617d5949cbba4baa2a6cfbb7e0a9b62c0/pybcj-1.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:746550dc7b5af4d04bb5fa4d065f18d39c925bcb5dee30db75747cd9a58bb6e8", size = 51665 }, + { url = "https://files.pythonhosted.org/packages/24/ea/da4637563468854bd361a69cd883946015f54fa119a5d9c655d26f151954/pybcj-1.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:8ce9b62b6aaa5b08773be8a919ecc4e865396c969f982b685eeca6e80c82abb7", size = 56041 }, + { url = "https://files.pythonhosted.org/packages/cf/b2/9b9e670818af925ed9a0168a5c021ccfcc089637d0e6651d16fd05896425/pybcj-1.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:493eab2b1f6f546730a6de0c5ceb75ce16f3767154e8ae30e2b70d41b928b7d2", size = 55606 }, + { url = "https://files.pythonhosted.org/packages/72/e9/d6b1bdf3a5aca8f3981145a5228ad51d72e2477a55927604a4768765e915/pybcj-1.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:ef55b96b7f2ed823e0b924de902065ec42ade856366c287dbb073fabd6b90ec1", size = 24719 }, +] + [[package]] name = "pycparser" version = "2.22" @@ -3875,6 +4087,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552 }, ] +[[package]] +name = "pycryptodomex" +version = "3.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/dc/e66551683ade663b5f07d7b3bc46434bf703491dbd22ee12d1f979ca828f/pycryptodomex-3.21.0.tar.gz", hash = "sha256:222d0bd05381dd25c32dd6065c071ebf084212ab79bab4599ba9e6a3e0009e6c", size = 4818543 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/5e/99f217d9881eead69607a2248dd7bbdf610837d7f5ad53f45a6cb71bbbfb/pycryptodomex-3.21.0-cp36-abi3-macosx_10_9_universal2.whl", hash = "sha256:34325b84c8b380675fd2320d0649cdcbc9cf1e0d1526edbe8fce43ed858cdc7e", size = 2499490 }, + { url = "https://files.pythonhosted.org/packages/ce/8f/4d0e2a859a6470289d64e39b419f01d2494dfa2e4995342d50f6c2834237/pycryptodomex-3.21.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:103c133d6cd832ae7266feb0a65b69e3a5e4dbbd6f3a3ae3211a557fd653f516", size = 1638037 }, + { url = "https://files.pythonhosted.org/packages/0c/9e/6e748c1fa814c956d356f93cf7192b19487ca56fc9e2a0bcde2bbc057601/pycryptodomex-3.21.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77ac2ea80bcb4b4e1c6a596734c775a1615d23e31794967416afc14852a639d3", size = 2172279 }, + { url = "https://files.pythonhosted.org/packages/46/3f/f5bef92b11750af9e3516d4e69736eeeff20a2818d34611508bef5a7b381/pycryptodomex-3.21.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9aa0cf13a1a1128b3e964dc667e5fe5c6235f7d7cfb0277213f0e2a783837cc2", size = 2258130 }, + { url = "https://files.pythonhosted.org/packages/de/4d/f0c65afd64ce435fd0547187ce6f99dfb37cdde16b05b57bca9f5c06966e/pycryptodomex-3.21.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:46eb1f0c8d309da63a2064c28de54e5e614ad17b7e2f88df0faef58ce192fc7b", size = 2297719 }, + { url = "https://files.pythonhosted.org/packages/1c/6a/2a1a101b0345ee70376ba93df8de6c8c01aac8341fda02970800873456a7/pycryptodomex-3.21.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:cc7e111e66c274b0df5f4efa679eb31e23c7545d702333dfd2df10ab02c2a2ce", size = 2164079 }, + { url = "https://files.pythonhosted.org/packages/3d/00/90a15f16c234815b660303c2d7266b41b401ea2605f3a90373e9d425e39f/pycryptodomex-3.21.0-cp36-abi3-musllinux_1_2_i686.whl", hash = "sha256:770d630a5c46605ec83393feaa73a9635a60e55b112e1fb0c3cea84c2897aa0a", size = 2333060 }, + { url = "https://files.pythonhosted.org/packages/61/74/49f5d20c514ccc631b940cc9dfec45dcce418dc84a98463a2e2ebec33904/pycryptodomex-3.21.0-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:52e23a0a6e61691134aa8c8beba89de420602541afaae70f66e16060fdcd677e", size = 2257982 }, + { url = "https://files.pythonhosted.org/packages/92/4b/d33ef74e2cc0025a259936661bb53432c5bbbadc561c5f2e023bcd73ce4c/pycryptodomex-3.21.0-cp36-abi3-win32.whl", hash = "sha256:a3d77919e6ff56d89aada1bd009b727b874d464cb0e2e3f00a49f7d2e709d76e", size = 1779052 }, + { url = "https://files.pythonhosted.org/packages/5b/be/7c991840af1184009fc86267160948350d1bf875f153c97bb471ad944e40/pycryptodomex-3.21.0-cp36-abi3-win_amd64.whl", hash = "sha256:b0e9765f93fe4890f39875e6c90c96cb341767833cfa767f41b490b506fa9ec0", size = 1816307 }, + { url = "https://files.pythonhosted.org/packages/af/ac/24125ad36778914a36f08d61ba5338cb9159382c638d9761ee19c8de822c/pycryptodomex-3.21.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:feaecdce4e5c0045e7a287de0c4351284391fe170729aa9182f6bd967631b3a8", size = 1694999 }, + { url = "https://files.pythonhosted.org/packages/93/73/be7a54a5903508070e5508925ba94493a1f326cfeecfff750e3eb250ea28/pycryptodomex-3.21.0-pp27-pypy_73-win32.whl", hash = "sha256:365aa5a66d52fd1f9e0530ea97f392c48c409c2f01ff8b9a39c73ed6f527d36c", size = 1769437 }, + { url = "https://files.pythonhosted.org/packages/e5/9f/39a6187f3986841fa6a9f35c6fdca5030ef73ff708b45a993813a51d7d10/pycryptodomex-3.21.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3efddfc50ac0ca143364042324046800c126a1d63816d532f2e19e6f2d8c0c31", size = 1619607 }, + { url = "https://files.pythonhosted.org/packages/f8/70/60bb08e9e9841b18d4669fb69d84b64ce900aacd7eb0ebebd4c7b9bdecd3/pycryptodomex-3.21.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0df2608682db8279a9ebbaf05a72f62a321433522ed0e499bc486a6889b96bf3", size = 1653571 }, + { url = "https://files.pythonhosted.org/packages/c9/6f/191b73509291c5ff0dddec9cc54797b1d73303c12b2e4017b24678e57099/pycryptodomex-3.21.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5823d03e904ea3e53aebd6799d6b8ec63b7675b5d2f4a4bd5e3adcb512d03b37", size = 1691548 }, + { url = "https://files.pythonhosted.org/packages/2d/c7/a0d3356f3074ac548afefa515ff46f3bea011deca607faf1c09b26dd5330/pycryptodomex-3.21.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:27e84eeff24250ffec32722334749ac2a57a5fd60332cd6a0680090e7c42877e", size = 1792099 }, +] + [[package]] name = "pydantic" version = "2.8.2" @@ -4093,6 +4329,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/5e/c86a5643653825d3c913719e788e41386bee415c2b87b4f955432f2de6b2/pypdf2-3.0.1-py3-none-any.whl", hash = "sha256:d16e4205cfee272fbdc0568b68d82be796540b1537508cef59388f839c191928", size = 232572 }, ] +[[package]] +name = "pyppmd" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/39/c8/9039c7503577de08a3f4c81e7619583efdc16030da6d1a25268d3dca49c8/pyppmd-1.1.0.tar.gz", hash = "sha256:1d38ce2e4b7eb84b53bc8a52380b94f66ba6c39328b8800b30c2b5bf31693973", size = 1348949 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/10/b19621035862e2ae12a1ba14c5b5c0a0befb27906bc00691642d7bdbdce6/pyppmd-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c5cd428715413fe55abf79dc9fc54924ba7e518053e1fc0cbdf80d0d99cf1442", size = 75756 }, + { url = "https://files.pythonhosted.org/packages/85/4a/a7c172cd431c4e1ddf9be349dc4bcfea81c2a236d2fe51bbfdcd697af55a/pyppmd-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0e96cc43f44b7658be2ea764e7fa99c94cb89164dbb7cdf209178effc2168319", size = 47347 }, + { url = "https://files.pythonhosted.org/packages/0d/32/f7357e0412e977ede4d63ba8bf55d014e5ea5b311818b2b0a1fee6d91baa/pyppmd-1.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dd20142869094bceef5ab0b160f4fff790ad1f612313a1e3393a51fc3ba5d57e", size = 46640 }, + { url = "https://files.pythonhosted.org/packages/b5/8e/1f416819f0aab17de47b15b72d0e9b05e2bf795c6e28d9f403ac01398b74/pyppmd-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a4f9b51e45c11e805e74ea6f6355e98a6423b5bbd92f45aceee24761bdc3d3b8", size = 135666 }, + { url = "https://files.pythonhosted.org/packages/73/ac/7d07d3ac6874f235554de392de08e6a369001db43cd6a619af4fbe02fb55/pyppmd-1.1.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:459f85e928fb968d0e34fb6191fd8c4e710012d7d884fa2b317b2e11faac7c59", size = 132892 }, + { url = "https://files.pythonhosted.org/packages/09/76/61db4268a439cfba8736b14130d928d199633fab2360a2c5043332a427d2/pyppmd-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f73cf2aaf60477eef17f5497d14b6099d8be9748390ad2b83d1c88214d050c05", size = 138901 }, + { url = "https://files.pythonhosted.org/packages/8b/9c/546729489ae07c0d7c2bfe37c69ae1cd3ce35a18ab000480ea4e8f12754f/pyppmd-1.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2ea3ae0e92c0b5345cd3a4e145e01bbd79c2d95355481ea5d833b5c0cb202a2d", size = 139725 }, + { url = "https://files.pythonhosted.org/packages/f7/db/4e734e97541554a389e7adb2a2a5c86ad8ae35c4dafe817b12fdc317de1a/pyppmd-1.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:775172c740133c0162a01c1a5443d0e312246881cdd6834421b644d89a634b91", size = 131598 }, + { url = "https://files.pythonhosted.org/packages/b1/8f/530e47290e07d2fdedfd345fc72af08226ccdd4cc913c2b895a8396c17b6/pyppmd-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:14421030f1d46f69829698bdd960698a3b3df0925e3c470e82cfcdd4446b7bc1", size = 142767 }, + { url = "https://files.pythonhosted.org/packages/a5/f9/16e0adfef500b171a96ed3c95f4a4d999f99cc79de3e415146808b19c2fb/pyppmd-1.1.0-cp310-cp310-win32.whl", hash = "sha256:b691264f9962532aca3bba5be848b6370e596d0a2ca722c86df388be08d0568a", size = 41283 }, + { url = "https://files.pythonhosted.org/packages/37/8d/c4846ab632e13ead87189f31bcc51fc825c75078d162a4a9dc8aed0a5b97/pyppmd-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:216b0d969a3f06e35fbfef979706d987d105fcb1e37b0b1324f01ee143719c4a", size = 46078 }, + { url = "https://files.pythonhosted.org/packages/27/0e/9db5d7c6ca3159aa0f07c0f1d5c59079176e7c57740a61aca62a39661178/pyppmd-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1f8c51044ee4df1b004b10bf6b3c92f95ea86cfe1111210d303dca44a56e4282", size = 75781 }, + { url = "https://files.pythonhosted.org/packages/f0/1b/4894b5c71feee76d3dfccf4383b59841f9bfd27aecf912b6542a2ab1e073/pyppmd-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ac25b3a13d1ac9b8f0bde46952e10848adc79d932f2b548a6491ef8825ae0045", size = 47370 }, + { url = "https://files.pythonhosted.org/packages/50/98/57b2c281e546f682279bd4a2577045d1f6d527c8fa2151a990b2a9bc48c2/pyppmd-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c8d3003eebe6aabe22ba744a38a146ed58a25633420d5da882b049342b7c8036", size = 46633 }, + { url = "https://files.pythonhosted.org/packages/06/72/b7e37aa69b7a105bcc119bc171437fbcb104aef2568b68ec8ed21a3fcdd1/pyppmd-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c520656bc12100aa6388df27dd7ac738577f38bf43f4a4bea78e1861e579ea5", size = 138233 }, + { url = "https://files.pythonhosted.org/packages/60/73/4f53a3c7730e1cba3f210b35ed6779e0fe302739196f43452664e079c0b5/pyppmd-1.1.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c2a3e807028159a705951f5cb5d005f94caed11d0984e59cc50506de543e22d", size = 135486 }, + { url = "https://files.pythonhosted.org/packages/31/7c/956ebf1f07506bb59e6f13ef068d91f1bec828758d399b455b175b668f6c/pyppmd-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec8a2447e69444703e2b273247bfcd4b540ec601780eff07da16344c62d2993d", size = 141183 }, + { url = "https://files.pythonhosted.org/packages/73/b4/4863499e012c555f4619dbebc5b83d79818e0161d9b6fb8b1e709fb1d6c7/pyppmd-1.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b9e0c8053e69cad6a92a0889b3324f567afc75475b4f54727de553ac4fc85780", size = 141752 }, + { url = "https://files.pythonhosted.org/packages/b4/cc/44e175222b31f86d0192d1d0d2c46c4bf0e933c9a06a65ff39596ad05666/pyppmd-1.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:5938d256e8d2a2853dc3af8bb58ae6b4a775c46fc891dbe1826a0b3ceb624031", size = 133921 }, + { url = "https://files.pythonhosted.org/packages/f1/d9/2f2e222d43ab274909e8dcd16d25cd4cc0245a8d59f93f8d6397cd4dc49f/pyppmd-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1ce5822d8bea920856232ccfb3c26b56b28b6846ea1b0eb3d5cb9592a026649e", size = 145191 }, + { url = "https://files.pythonhosted.org/packages/6d/e7/1214571442624e2314ed1ed5ba0081358335fc760fb455c3d8df83b118c6/pyppmd-1.1.0-cp311-cp311-win32.whl", hash = "sha256:2a9e894750f2a52b03e3bc0d7cf004d96c3475a59b1af7e797d808d7d29c9ffe", size = 41286 }, + { url = "https://files.pythonhosted.org/packages/8e/7f/d3cc8443bd2b56bc54ea205dcf73d70ef8d4342096ff33fc8719956f45e9/pyppmd-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:969555c72e72fe2b4dd944127521a8f2211caddb5df452bbc2506b5adfac539e", size = 46087 }, + { url = "https://files.pythonhosted.org/packages/bf/0b/4c8e3a92c4366a9aa2d801ab4bd7ba72bd1d214da890dd91ab4d73e52878/pyppmd-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9d6ef8fd818884e914bc209f7961c9400a4da50d178bba25efcef89f09ec9169", size = 76116 }, + { url = "https://files.pythonhosted.org/packages/e1/0b/45fdf5a28c810ed4d3c0cb05ae5346e2972cdbfe89f374b263e07c5b820d/pyppmd-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:95f28e2ecf3a9656bd7e766aaa1162b6872b575627f18715f8b046e8617c124a", size = 47633 }, + { url = "https://files.pythonhosted.org/packages/56/a4/4aa1d36d98f3786c8b12ac96ac8234d7dc3c2a9e8f5174a5698f424099ec/pyppmd-1.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:37f3557ea65ee417abcdf5f49d35df00bb9f6f252639cae57aeefcd0dd596133", size = 46704 }, + { url = "https://files.pythonhosted.org/packages/d9/70/a49389a6666f670db5ecc7caa37030c9a9abfeea455c387172584551a271/pyppmd-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e84b25d088d7727d50218f57f92127cdb839acd6ec3de670b6680a4cf0b2d2a", size = 139145 }, + { url = "https://files.pythonhosted.org/packages/30/4c/f08cdf618744a3cce0da106ecf6e427b24d27b0bb1484afc40b88ca23a39/pyppmd-1.1.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:99ed42891986dac8c2ecf52bddfb777900233d867aa18849dbba6f3335600466", size = 136618 }, + { url = "https://files.pythonhosted.org/packages/bb/e0/afc0fb971c893e9e72cc8d70df93c50b3f3ebb12b4bdb21f869b775faf7e/pyppmd-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6fe69b82634488ada75ba07efb90cd5866fa3d64a2c12932b6e8ae207a14e5f", size = 142757 }, + { url = "https://files.pythonhosted.org/packages/26/b2/793e92c7a66de0b0b8d777c3c4df3ee5a5bec7fbaf0b69ab7374cefefa43/pyppmd-1.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:60981ffde1fe6ade750b690b35318c41a1160a8505597fda2c39a74409671217", size = 142749 }, + { url = "https://files.pythonhosted.org/packages/5e/6e/a1bf750bc7ed025a06600c65917d02e3c6dea7dfa728746c7251d4910d37/pyppmd-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:46e8240315476f57aac23d71e6de003e122b65feba7c68f4cc46a089a82a7cd4", size = 135033 }, + { url = "https://files.pythonhosted.org/packages/1e/ee/4a12a4b1990f1fabb77f9ef94d2cd6c795690eec79ad135b8236dc59dbd2/pyppmd-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c0308e2e76ecb4c878a18c2d7a7c61dbca89b4ef138f65d5f5ead139154dcdea", size = 146510 }, + { url = "https://files.pythonhosted.org/packages/04/cd/a6571420345315f5340ac10897726303ae07260cb025dc4a60371d1e8b97/pyppmd-1.1.0-cp312-cp312-win32.whl", hash = "sha256:b4fa4c27dc1314d019d921f2aa19e17f99250557e7569eeb70e180558f46af74", size = 41332 }, + { url = "https://files.pythonhosted.org/packages/c0/a4/af77129d671d6adcc6c82e1b0f03f0ad0b70c44ac70ed4c72b5c8952553b/pyppmd-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:c269d21e15f4175df27cf00296476097af76941f948734c642d7fb6e85b9b3b9", size = 46193 }, + { url = "https://files.pythonhosted.org/packages/6a/e2/1d5fbd6dde1234b635000072c8d1d87c7ed3acf01a3c4aa8082504d58bc5/pyppmd-1.1.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ad5da9f7592158e6b6b51d7cd15e536d8b23afbb4d22cba4e5744c7e0a3548b1", size = 41505 }, + { url = "https://files.pythonhosted.org/packages/24/66/9215c5dda61b3aa3259902a586dacd198b4b0793ab99228734091b5e7fa7/pyppmd-1.1.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc6543e7d12ef0a1466d291d655e3d6bca59c7336dbb53b62ccdd407822fb52b", size = 44814 }, + { url = "https://files.pythonhosted.org/packages/1a/87/cc2aa429688f238ae30f26b8334194a21e25643d3257c9e5b14cccdc578e/pyppmd-1.1.0-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a5e4008a45910e3c8c227f6f240de67eb14454c015dc3d8060fc41e230f395d3", size = 43629 }, + { url = "https://files.pythonhosted.org/packages/9f/96/cd3f64f6bdce091ffb6d2c1c23dc91e8b94e312a5d08cd648625555fb69e/pyppmd-1.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9301fa39d1fb0ed09a10b4c5d7f0074113e96a1ead16ba7310bedf95f7ef660c", size = 43911 }, + { url = "https://files.pythonhosted.org/packages/e6/ab/02ab90e2dddf2dd55e30e64fa0509627c6e0c86b26503a6df95ae55b1e45/pyppmd-1.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:59521a3c6028da0cb5780ba16880047b00163432a6b975da2f6123adfc1b0be8", size = 42427 }, +] + [[package]] name = "pyproj" version = "3.6.1" @@ -4122,6 +4404,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/f3/2f32fe143cd7ba1d4d68f1b6dce9ca402d909cbd5a5830e3a8fa3d1acbbf/pyproj-3.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:7a27151ddad8e1439ba70c9b4b2b617b290c39395fa9ddb7411ebb0eb86d6fb0", size = 6079779 }, ] +[[package]] +name = "pyreadr" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pandas" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ab/9b/732136124fec08851ce846544b3d48e9f109638abd250f6c0311671db636/pyreadr-0.5.2.tar.gz", hash = "sha256:33d5747fe210d41e4a329afef2232c9d1258847094b87a2a96efffbef223fae8", size = 1175696 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/f5/087d0ea21bf31f43f2ca377f5f03b15e43fc406cd7e1fcfc73a9f4fc055e/pyreadr-0.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ab20da5c1b1a547316bd45f0a8984f3c7c26bb548a41c8c606da939766ae3bc9", size = 304689 }, + { url = "https://files.pythonhosted.org/packages/37/36/d394c0bd09c37e24e9a0435989ee1c5fe91470a0faf43475eb7a41493f68/pyreadr-0.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0b445a6ffe09922ade3dc003585a589163a5dbab2d006aebcd2e00c7738e17c0", size = 300820 }, + { url = "https://files.pythonhosted.org/packages/fa/3b/af53ad8bebc3aa9252ccc99d1394fa85c185daa34aac0295a286f58d6f29/pyreadr-0.5.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5552b17a2e32b885b789a50bfdf895663fb6fc2e0db85351b620a74ece474c6", size = 415788 }, + { url = "https://files.pythonhosted.org/packages/ca/69/0a6abc458561f7f84ee87ac2b872ea4cd355c376822e73ebc7d44ff94536/pyreadr-0.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35967860dace4a736108740b6127abb59c2e25c1b081f426181907666c7f9dac", size = 416973 }, + { url = "https://files.pythonhosted.org/packages/2a/00/9141eb7747cd6a04425f071adfb32226d77094766d9901b856345d832743/pyreadr-0.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:fae6a270842bd5d0d2729ab2ad6bb0125786b0b1023fd3cdc51deedc1d66cb61", size = 1251181 }, + { url = "https://files.pythonhosted.org/packages/cf/47/1f666551444156b18524632fb93da10bbcf9a9cf0cb20faebd1f85512558/pyreadr-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5b8fa9c92609e2b925900ba1e5f236a9d38bfd49ad37038e41ac1c7d853fddda", size = 301435 }, + { url = "https://files.pythonhosted.org/packages/9e/7a/f9b6786876cbc016019d7de7fef5ed7a5397fbb986cfa9f10fdbef52ec03/pyreadr-0.5.2-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:49b94459a1ed0003e59861f0ad088dc17f97632e451797b79cbf12fe312e6b6d", size = 305194 }, + { url = "https://files.pythonhosted.org/packages/8b/72/73b2979ee06d58844a4f7206dd96c4927a4406c15645945d69c90e00c968/pyreadr-0.5.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5698418c4fedcbc0b7600c967a2943a79803d39cf7f13b73999c58815f7a5ebc", size = 414793 }, + { url = "https://files.pythonhosted.org/packages/79/9f/a0e84861e0d0939b71830d30960c9215487d5278d2d89890988035f02b9c/pyreadr-0.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2af40aa8de034c24cdefd2dfdbf8c21277cf1c27cce063729604e48aa908aba", size = 416025 }, + { url = "https://files.pythonhosted.org/packages/61/c3/a5eb51a2c8d65c78da160618c6d892178aadf3b977928994efe6a27a5f8c/pyreadr-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:82e1a5388d8910eac75e3cd37696851c52175a0b263afaedd570e000d385781c", size = 1251639 }, + { url = "https://files.pythonhosted.org/packages/58/5f/e6c0726bb90c2bb7371d7b68b825c03aaf6bfc89d35b88844a4979de0767/pyreadr-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7750624a6158ba0393c0cbc7f44a169735799862089891408f9a83ece04662bf", size = 300822 }, + { url = "https://files.pythonhosted.org/packages/c0/b0/7e955064875988eb000286e405952f1b0931a366f15303e78abcb8a2256a/pyreadr-0.5.2-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:f0c8feaa26f5f45f91bfc07dbea7334c7c22cfaf3fa923e203f30a5c976eda71", size = 303775 }, + { url = "https://files.pythonhosted.org/packages/8a/74/b9c6b39170e7272ebdb44091dce6ad07181bce465becd22977e85d530b9a/pyreadr-0.5.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3f200d4bd7dcfa37b9d244f05b3708b7183bb7978c6865a1364f39727021fb0", size = 412713 }, + { url = "https://files.pythonhosted.org/packages/10/54/1cd3de1f08d8f86bf273de23d55a1b25e4dccaae45be116512a7b2cfff0b/pyreadr-0.5.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3aa4919beb4cfa9c915e704b41d32d8c40a4f505a7c9bfdfc4930b3b4da5d2b8", size = 413567 }, + { url = "https://files.pythonhosted.org/packages/bb/2b/e16e4acc1bacde128add8ae3596f2cd0d899cb8d304332700827fd18c5f8/pyreadr-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:6bae17a8970b62c2af257ec8c5aad9d759a1bdc2a763e299ff82826d7140afe4", size = 1249250 }, +] + [[package]] name = "pyright" version = "1.1.373" @@ -4385,6 +4693,62 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/d2/3b2ab40f455a256cb6672186bea95cd97b459ce4594050132d71e76f0d6f/pyzmq-26.2.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:90412f2db8c02a3864cbfc67db0e3dcdbda336acf1c469526d3e869394fe001c", size = 550762 }, ] +[[package]] +name = "pyzstd" +version = "0.16.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/62/14/878fee4072cecb1cc6e061c7d0d933e481389c27de939538c9cc3f18894a/pyzstd-0.16.2.tar.gz", hash = "sha256:179c1a2ea1565abf09c5f2fd72f9ce7c54b2764cf7369e05c0bfd8f1f67f63d2", size = 789505 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/a9/efad061c5a982f859ba8bf5de565d73567f87ad8bba3364fe28e9a8672b6/pyzstd-0.16.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:637376c8f8cbd0afe1cab613f8c75fd502bd1016bf79d10760a2d5a00905fe62", size = 372191 }, + { url = "https://files.pythonhosted.org/packages/b6/36/eb6dcfacb273ca13dfa20d296f27ffd0a6c53677965f868625edf764b71e/pyzstd-0.16.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3e7a7118cbcfa90ca2ddbf9890c7cb582052a9a8cf2b7e2c1bbaf544bee0f16a", size = 295083 }, + { url = "https://files.pythonhosted.org/packages/fb/76/a7862487402123f221439808ed50915e00cfc8e1df7365af366610176347/pyzstd-0.16.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a74cb1ba05876179525144511eed3bd5a509b0ab2b10632c1215a85db0834dfd", size = 390166 }, + { url = "https://files.pythonhosted.org/packages/b8/52/1e1ab63026d67f18b9841285576d59bb799b838a5de4f852ad9e054674a1/pyzstd-0.16.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c084dde218ffbf112e507e72cbf626b8f58ce9eb23eec129809e31037984662", size = 472043 }, + { url = "https://files.pythonhosted.org/packages/0d/24/14c8948b9d16d399ff80504bc404bb091b0eb5339f6fbdad0481da751c09/pyzstd-0.16.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4646459ebd3d7a59ddbe9312f020bcf7cdd1f059a2ea07051258f7af87a0b31", size = 415258 }, + { url = "https://files.pythonhosted.org/packages/6b/3e/e4c7f449af9d19975ff5d333a58330317cf8b05fe4754106c694a29e7c25/pyzstd-0.16.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14bfc2833cc16d7657fc93259edeeaa793286e5031b86ca5dc861ba49b435fce", size = 413680 }, + { url = "https://files.pythonhosted.org/packages/10/09/8918853028cf593c141456b9a42d68420beec3f16a8cc4f1aa5d0b8b0c84/pyzstd-0.16.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f27d488f19e5bf27d1e8aa1ae72c6c0a910f1e1ffbdf3c763d02ab781295dd27", size = 412630 }, + { url = "https://files.pythonhosted.org/packages/47/20/5a4c899530571e0e8ecdcb9dc7e3fc38491d4b342fbd7d8413805c88013b/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:91e134ca968ff7dcfa8b7d433318f01d309b74ee87e0d2bcadc117c08e1c80db", size = 404980 }, + { url = "https://files.pythonhosted.org/packages/0a/1d/aeeeebb702d3500a01b5b1029ba1716aea3afa75e8aacb904806b3f1afe5/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:6b5f64cd3963c58b8f886eb6139bb8d164b42a74f8a1bb95d49b4804f4592d61", size = 418000 }, + { url = "https://files.pythonhosted.org/packages/fc/0c/66ca36d24ad97af40a8fe8de9e3f316a5f4fd2fb3cab8634a2f7da5571c8/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:0b4a8266871b9e0407f9fd8e8d077c3558cf124d174e6357b523d14f76971009", size = 485576 }, + { url = "https://files.pythonhosted.org/packages/39/66/6c1de1347de94aa85f60e854cccae0948bda2eda2351e4d47c8bb0a7cf18/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:1bb19f7acac30727354c25125922aa59f44d82e0e6a751df17d0d93ff6a73853", size = 564542 }, + { url = "https://files.pythonhosted.org/packages/6d/46/75365a3ab279d58e69d410ce0a21527e689fa651837227e23dee294d096f/pyzstd-0.16.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3008325b7368e794d66d4d98f2ee1d867ef5afd09fd388646ae02b25343c420d", size = 430619 }, + { url = "https://files.pythonhosted.org/packages/0d/62/17bf81d42acbd39bffdea559b6fbd7ec331cd74bc52f249e536fefe5480d/pyzstd-0.16.2-cp310-cp310-win32.whl", hash = "sha256:66f2d5c0bbf5bf32c577aa006197b3525b80b59804450e2c32fbcc2d16e850fd", size = 218224 }, + { url = "https://files.pythonhosted.org/packages/f7/b6/281245890df08a567186c6e262c43d68581291cca107c8d7304c37708e46/pyzstd-0.16.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fe5f5459ebe1161095baa7a86d04ab625b35148f6c425df0347ed6c90a2fd58", size = 245012 }, + { url = "https://files.pythonhosted.org/packages/10/5a/19d7aec81853f6dc53eabad388227e3beecfaca4788af23b8807a0ea2112/pyzstd-0.16.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1bdbe7f01c7f37d5cd07be70e32a84010d7dfd6677920c0de04cf7d245b60d", size = 372192 }, + { url = "https://files.pythonhosted.org/packages/29/35/2eb025e6a0fff49b5de8bea20e82e4d7d5456e634bf3809123fbe5e5f194/pyzstd-0.16.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1882a3ceaaf9adc12212d587d150ec5e58cfa9a765463d803d739abbd3ac0f7a", size = 295084 }, + { url = "https://files.pythonhosted.org/packages/04/1f/03785d7ff1ce73b9347533f798cb27afa57768e66012f97b18b7b7303158/pyzstd-0.16.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea46a8b9d60f6a6eba29facba54c0f0d70328586f7ef0da6f57edf7e43db0303", size = 390167 }, + { url = "https://files.pythonhosted.org/packages/b7/59/e307622115a2df30075efbd28933dc0ad8f2007c5ba5a3eb49c956de3d56/pyzstd-0.16.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d7865bc06589cdcecdede0deefe3da07809d5b7ad9044c224d7b2a0867256957", size = 472038 }, + { url = "https://files.pythonhosted.org/packages/97/21/870fda5454240089e9c37625320580d392b03beaeae4889c67c0a21c4d34/pyzstd-0.16.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:52f938a65b409c02eb825e8c77fc5ea54508b8fc44b5ce226db03011691ae8cc", size = 415217 }, + { url = "https://files.pythonhosted.org/packages/3c/35/b33faeeb9c96fddd08bf7871c9f5c4638c32ad79227155922fd4a63190c5/pyzstd-0.16.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e97620d3f53a0282947304189deef7ca7f7d0d6dfe15033469dc1c33e779d5e5", size = 413714 }, + { url = "https://files.pythonhosted.org/packages/aa/a3/b9058dd43eb52025a2ca78946dcb9ef9d8984acac172a698bcf12712217c/pyzstd-0.16.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7c40e9983d017108670dc8df68ceef14c7c1cf2d19239213274783041d0e64c", size = 412568 }, + { url = "https://files.pythonhosted.org/packages/12/31/fe7d462c912f2040775bfa2af4327f9fcebb16e8fa9c3bfa058bc1306722/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7cd4b3b2c6161066e4bde6af1cf78ed3acf5d731884dd13fdf31f1db10830080", size = 404988 }, + { url = "https://files.pythonhosted.org/packages/48/4c/582aca0e5210436499bce1639a8d15da3f76f8d5827da1aa3eeb2c4e271c/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:454f31fd84175bb203c8c424f2255a343fa9bd103461a38d1bf50487c3b89508", size = 417961 }, + { url = "https://files.pythonhosted.org/packages/39/e9/54f53641ff10b4ea18d3ba159b03bd07e6ae5a5b7ae01f1329b0c35b8ca2/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:5ef754a93743f08fb0386ce3596780bfba829311b49c8f4107af1a4bcc16935d", size = 485587 }, + { url = "https://files.pythonhosted.org/packages/ce/65/25243b3fea9e52a20bfece1b12e3d3ee3125f17b1735aab08cb9a7a760b4/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:be81081db9166e10846934f0e3576a263cbe18d81eca06e6a5c23533f8ce0dc6", size = 564543 }, + { url = "https://files.pythonhosted.org/packages/3b/3c/324b8ddca55b4b073b413cea3e0587af3c8153ccf7d6d63ed294831f2095/pyzstd-0.16.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:738bcb2fa1e5f1868986f5030955e64de53157fa1141d01f3a4daf07a1aaf644", size = 430628 }, + { url = "https://files.pythonhosted.org/packages/db/a1/aca18925e23bceb833fc742ebaf87aa9d1ba8b178f0332bd108fc8966482/pyzstd-0.16.2-cp311-cp311-win32.whl", hash = "sha256:0ea214c9b97046867d1657d55979021028d583704b30c481a9c165191b08d707", size = 218215 }, + { url = "https://files.pythonhosted.org/packages/c0/7f/0f5d1d1891e6c6e14d846d2881a06ab7e5e97cabeb5e1e9e53debec4091a/pyzstd-0.16.2-cp311-cp311-win_amd64.whl", hash = "sha256:c17c0fc02f0e75b0c7cd21f8eaf4c6ce4112333b447d93da1773a5f705b2c178", size = 245055 }, + { url = "https://files.pythonhosted.org/packages/28/15/20046759d138733e7150afa6aa15f322022d7587968e2dbd5b36fbf8aa86/pyzstd-0.16.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d4081fd841a9efe9ded7290ee7502dbf042c4158b90edfadea3b8a072c8ec4e1", size = 373230 }, + { url = "https://files.pythonhosted.org/packages/51/8d/55b536edaecf19d2f8dbd8fbaefd184f2f9cc6b71d241caa6d86bed96813/pyzstd-0.16.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fd3fa45d2aeb65367dd702806b2e779d13f1a3fa2d13d5ec777cfd09de6822de", size = 295699 }, + { url = "https://files.pythonhosted.org/packages/11/14/086e7f690154c6f3d9bdb46da26a4cd3c9e0b284346ce10943711ca48c32/pyzstd-0.16.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8b5f0d2c07994a5180d8259d51df6227a57098774bb0618423d7eb4a7303467", size = 390556 }, + { url = "https://files.pythonhosted.org/packages/90/d2/c6d854705d6fa0ad876209b4ba796ab31d85b710d1459029f2cb41085a8d/pyzstd-0.16.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60c9d25b15c7ae06ed5d516d096a0d8254f9bed4368b370a09cccf191eaab5cb", size = 472928 }, + { url = "https://files.pythonhosted.org/packages/aa/38/f97dd871e446adc834349caa605dbaf5bac86763a255f62c809cc2459c85/pyzstd-0.16.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:29acf31ce37254f6cad08deb24b9d9ba954f426fa08f8fae4ab4fdc51a03f4ae", size = 416057 }, + { url = "https://files.pythonhosted.org/packages/53/be/0c5ad7bf29dc890f6a3303760b9802aeeafa4e3ffb598de625f501986bfe/pyzstd-0.16.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ec77612a17697a9f7cf6634ffcee616eba9b997712fdd896e77fd19ab3a0618", size = 414613 }, + { url = "https://files.pythonhosted.org/packages/1f/1a/d3a1edcd59e2f62a35ac6257d2b86a2c872ae9a8e925380620a8db0d9a9a/pyzstd-0.16.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:313ea4974be93be12c9a640ab40f0fc50a023178aae004a8901507b74f190173", size = 413236 }, + { url = "https://files.pythonhosted.org/packages/f2/8d/912430c2310466c14a89a5a529b72eddef7e73fa733806dbe0b030cf3495/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e91acdefc8c2c6c3b8d5b1b5fe837dce4e591ecb7c0a2a50186f552e57d11203", size = 405536 }, + { url = "https://files.pythonhosted.org/packages/9e/83/4edb419a13b9d1e1debc01e88084eba93a5f7c10ef198da11f6782857c73/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:929bd91a403539e72b5b5cb97f725ac4acafe692ccf52f075e20cd9bf6e5493d", size = 419145 }, + { url = "https://files.pythonhosted.org/packages/8f/e9/62a169eddc37aefac480ee3b3318c221f6731e1e342dafd9e05b7fdaa7c5/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:740837a379aa32d110911ebcbbc524f9a9b145355737527543a884bd8777ca4f", size = 487157 }, + { url = "https://files.pythonhosted.org/packages/57/9d/5949f2a0144d1f99fab7914f854b582d2784c73139cc190e603e4d6b7b37/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:adfc0e80dd157e6d1e0b0112c8ecc4b58a7a23760bd9623d74122ef637cfbdb6", size = 565918 }, + { url = "https://files.pythonhosted.org/packages/de/ce/647b9c7602ac477c9e62cf9399810f72bb5dba8f508e7cdf8be1d260e6f9/pyzstd-0.16.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:79b183beae1c080ad3dca39019e49b7785391947f9aab68893ad85d27828c6e7", size = 431373 }, + { url = "https://files.pythonhosted.org/packages/8b/fb/4141e3d4549eea26e5a59ec723eade271980816cb2ed7613df855baa672f/pyzstd-0.16.2-cp312-cp312-win32.whl", hash = "sha256:b8d00631a3c466bc313847fab2a01f6b73b3165de0886fb03210e08567ae3a89", size = 218541 }, + { url = "https://files.pythonhosted.org/packages/51/b9/e1373b179129c2095d70bd1df02a51d388f4c7e4ecb62acb4e5e9570269b/pyzstd-0.16.2-cp312-cp312-win_amd64.whl", hash = "sha256:c0d43764e9a60607f35d8cb3e60df772a678935ab0e02e2804d4147377f4942c", size = 245320 }, + { url = "https://files.pythonhosted.org/packages/f9/ad/c09fb722c12a82b826c97efc50a919e229bfbaf644f5a140adcd71941473/pyzstd-0.16.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4b631117b97a42ff6dfd0ffc885a92fff462d7c34766b28383c57b996f863338", size = 364187 }, + { url = "https://files.pythonhosted.org/packages/57/f9/93175fe72f85fb675fe04abca296fe583112a25d0ec7faa026288d9463c2/pyzstd-0.16.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:56493a3fbe1b651a02102dd0902b0aa2377a732ff3544fb6fb3f114ca18db52f", size = 279825 }, + { url = "https://files.pythonhosted.org/packages/8a/de/0b40acf76d7ed1f7975877535e004de85ec2e869632754b5d4d389258b8a/pyzstd-0.16.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1eae9bdba4a1e5d3181331f403114ff5b8ce0f4b569f48eba2b9beb2deef1e4", size = 321313 }, + { url = "https://files.pythonhosted.org/packages/41/5e/00102bacd1a7c957c88098f3ae2cdac17842ac0f94d2e685ff5b75a05730/pyzstd-0.16.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1be6972391c8aeecc7e61feb96ffc8e77a401bcba6ed994e7171330c45a1948", size = 344376 }, + { url = "https://files.pythonhosted.org/packages/a3/95/27a7da3dbd4460cd9432bdc22d9d5f8ec77c86275d069020fa74ea280f7f/pyzstd-0.16.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:761439d687e3a5687c2ff5c6a1190e1601362a4a3e8c6c82ff89719d51d73e19", size = 328591 }, + { url = "https://files.pythonhosted.org/packages/c2/03/8f4d5fd45f6bfad66d67cdf583492a9f52a21049f60e6b36a7e9f8aa7adc/pyzstd-0.16.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f5fbdb8cf31b60b2dc586fecb9b73e2f172c21a0b320ed275f7b8d8a866d9003", size = 240786 }, +] + [[package]] name = "qtconsole" version = "5.6.0" @@ -4831,26 +5195,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.1.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/25/4c/2f786388acd82c295eedc4afeede7ef4b29cf27277151d8d13be906bac70/ruff-0.1.6.tar.gz", hash = "sha256:1b09f29b16c6ead5ea6b097ef2764b42372aebe363722f1605ecbcd2b9207184", size = 1719627 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/75/5054ec93ec0d5db26e218cb2814ddaa085ba1f29fad0ec56dd8107a97688/ruff-0.1.6-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:88b8cdf6abf98130991cbc9f6438f35f6e8d41a02622cc5ee130a02a0ed28703", size = 11628468 }, - { url = "https://files.pythonhosted.org/packages/a2/91/8b2920f6026c069ae0802fc3c44f7337e04bf2a198ce94bfab360073477a/ruff-0.1.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5c549ed437680b6105a1299d2cd30e4964211606eeb48a0ff7a93ef70b902248", size = 5990066 }, - { url = "https://files.pythonhosted.org/packages/df/1e/03ef0cc5c7d03e50d4f954218551d6001f1f70e6f391cdb678efb5c6e6ab/ruff-0.1.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cf5f701062e294f2167e66d11b092bba7af6a057668ed618a9253e1e90cfd76", size = 5929313 }, - { url = "https://files.pythonhosted.org/packages/92/7c/38fd1b9cb624f5725a6a08c81bf7e823c64b28622ffcb4369c56dc0a16d0/ruff-0.1.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:05991ee20d4ac4bb78385360c684e4b417edd971030ab12a4fbd075ff535050e", size = 5627943 }, - { url = "https://files.pythonhosted.org/packages/c7/c3/98e3d0eb92e5a2ec10f76c71067640b6f21def23c3b1ff8f08ab6348255e/ruff-0.1.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87455a0c1f739b3c069e2f4c43b66479a54dea0276dd5d4d67b091265f6fd1dc", size = 6074206 }, - { url = "https://files.pythonhosted.org/packages/e8/33/62fb966eb70d9bb45ddf5023d40e26946a5e5127d99956b84c8a9a76b153/ruff-0.1.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:683aa5bdda5a48cb8266fcde8eea2a6af4e5700a392c56ea5fb5f0d4bfdc0240", size = 6743205 }, - { url = "https://files.pythonhosted.org/packages/c7/f1/60d43182f98113156a1b21a17f30541dda9f5ffcfeedc2b54dc030a2c413/ruff-0.1.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:137852105586dcbf80c1717facb6781555c4e99f520c9c827bd414fac67ddfb6", size = 6600581 }, - { url = "https://files.pythonhosted.org/packages/09/92/36850598e84f75cfe8edd252dbf40442b4cc226ed2c76206a9b3cbfb9986/ruff-0.1.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd98138a98d48a1c36c394fd6b84cd943ac92a08278aa8ac8c0fdefcf7138f35", size = 7708098 }, - { url = "https://files.pythonhosted.org/packages/3b/2f/8ef67614631622aa3ea79b27e01ac86d7f90a988520454e3a84cb2fd890f/ruff-0.1.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a0cd909d25f227ac5c36d4e7e681577275fb74ba3b11d288aff7ec47e3ae745", size = 6295145 }, - { url = "https://files.pythonhosted.org/packages/3c/4b/af366db98d15efe83fd3e3aae7319d3897e3475fc53a2f1b0287c8255422/ruff-0.1.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e8fd1c62a47aa88a02707b5dd20c5ff20d035d634aa74826b42a1da77861b5ff", size = 5878111 }, - { url = "https://files.pythonhosted.org/packages/bf/af/25b794e750f1d74a83ce6b16625e3306beeb2161c517b9d883958de05526/ruff-0.1.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fd89b45d374935829134a082617954120d7a1470a9f0ec0e7f3ead983edc48cc", size = 5629339 }, - { url = "https://files.pythonhosted.org/packages/81/b0/92c4cb6bceb19ebd27cedd1f45b337f7fd5397e6b760094831266be59661/ruff-0.1.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:491262006e92f825b145cd1e52948073c56560243b55fb3b4ecb142f6f0e9543", size = 5959772 }, - { url = "https://files.pythonhosted.org/packages/11/02/3a7e3101d88b113f326e0fdf3f566fba2600fc4b1fd828d56027d293e22d/ruff-0.1.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ea284789861b8b5ca9d5443591a92a397ac183d4351882ab52f6296b4fdd5462", size = 6356759 }, - { url = "https://files.pythonhosted.org/packages/fc/93/8a4b3523c4564168aab720d8361b4bf12173b099509caaac93168b72927a/ruff-0.1.6-py3-none-win32.whl", hash = "sha256:1610e14750826dfc207ccbcdd7331b6bd285607d4181df9c1c6ae26646d6848a", size = 5835632 }, - { url = "https://files.pythonhosted.org/packages/40/e3/55ae013087bd892dd61c25895a119ffec2f9b9f7c3d1fd0cea5f5f7bd74a/ruff-0.1.6-py3-none-win_amd64.whl", hash = "sha256:4558b3e178145491e9bc3b2ee3c4b42f19d19384eaa5c59d10acf6e8f8b57e33", size = 6226075 }, - { url = "https://files.pythonhosted.org/packages/21/f7/fdce733e594e1b274fc9232256be6a4e03e8d2cb9e354c783801191e2f4d/ruff-0.1.6-py3-none-win_arm64.whl", hash = "sha256:03910e81df0d8db0e30050725a5802441c2022ea3ae4fe0609b76081731accbc", size = 5965823 }, +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/2b/01245f4f3a727d60bebeacd7ee6d22586c7f62380a2597ddb22c2f45d018/ruff-0.8.2.tar.gz", hash = "sha256:b84f4f414dda8ac7f75075c1fa0b905ac0ff25361f42e6d5da681a465e0f78e5", size = 3349020 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/29/366be70216dba1731a00a41f2f030822b0c96c7c4f3b2c0cdce15cbace74/ruff-0.8.2-py3-none-linux_armv6l.whl", hash = "sha256:c49ab4da37e7c457105aadfd2725e24305ff9bc908487a9bf8d548c6dad8bb3d", size = 10530649 }, + { url = "https://files.pythonhosted.org/packages/63/82/a733956540bb388f00df5a3e6a02467b16c0e529132625fe44ce4c5fb9c7/ruff-0.8.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ec016beb69ac16be416c435828be702ee694c0d722505f9c1f35e1b9c0cc1bf5", size = 10274069 }, + { url = "https://files.pythonhosted.org/packages/3d/12/0b3aa14d1d71546c988a28e1b412981c1b80c8a1072e977a2f30c595cc4a/ruff-0.8.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:f05cdf8d050b30e2ba55c9b09330b51f9f97d36d4673213679b965d25a785f3c", size = 9909400 }, + { url = "https://files.pythonhosted.org/packages/23/08/f9f08cefb7921784c891c4151cce6ed357ff49e84b84978440cffbc87408/ruff-0.8.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:60f578c11feb1d3d257b2fb043ddb47501ab4816e7e221fbb0077f0d5d4e7b6f", size = 10766782 }, + { url = "https://files.pythonhosted.org/packages/e4/71/bf50c321ec179aa420c8ec40adac5ae9cc408d4d37283a485b19a2331ceb/ruff-0.8.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cbd5cf9b0ae8f30eebc7b360171bd50f59ab29d39f06a670b3e4501a36ba5897", size = 10286316 }, + { url = "https://files.pythonhosted.org/packages/f2/83/c82688a2a6117539aea0ce63fdf6c08e60fe0202779361223bcd7f40bd74/ruff-0.8.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b402ddee3d777683de60ff76da801fa7e5e8a71038f57ee53e903afbcefdaa58", size = 11338270 }, + { url = "https://files.pythonhosted.org/packages/7f/d7/bc6a45e5a22e627640388e703160afb1d77c572b1d0fda8b4349f334fc66/ruff-0.8.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:705832cd7d85605cb7858d8a13d75993c8f3ef1397b0831289109e953d833d29", size = 12058579 }, + { url = "https://files.pythonhosted.org/packages/da/3b/64150c93946ec851e6f1707ff586bb460ca671581380c919698d6a9267dc/ruff-0.8.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:32096b41aaf7a5cc095fa45b4167b890e4c8d3fd217603f3634c92a541de7248", size = 11615172 }, + { url = "https://files.pythonhosted.org/packages/e4/9e/cf12b697ea83cfe92ec4509ae414dc4c9b38179cc681a497031f0d0d9a8e/ruff-0.8.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e769083da9439508833cfc7c23e351e1809e67f47c50248250ce1ac52c21fb93", size = 12882398 }, + { url = "https://files.pythonhosted.org/packages/a9/27/96d10863accf76a9c97baceac30b0a52d917eb985a8ac058bd4636aeede0/ruff-0.8.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fe716592ae8a376c2673fdfc1f5c0c193a6d0411f90a496863c99cd9e2ae25d", size = 11176094 }, + { url = "https://files.pythonhosted.org/packages/eb/10/cd2fd77d4a4e7f03c29351be0f53278a393186b540b99df68beb5304fddd/ruff-0.8.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:81c148825277e737493242b44c5388a300584d73d5774defa9245aaef55448b0", size = 10771884 }, + { url = "https://files.pythonhosted.org/packages/71/5d/beabb2ff18870fc4add05fa3a69a4cb1b1d2d6f83f3cf3ae5ab0d52f455d/ruff-0.8.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d261d7850c8367704874847d95febc698a950bf061c9475d4a8b7689adc4f7fa", size = 10382535 }, + { url = "https://files.pythonhosted.org/packages/ae/29/6b3fdf3ad3e35b28d87c25a9ff4c8222ad72485ab783936b2b267250d7a7/ruff-0.8.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1ca4e3a87496dc07d2427b7dd7ffa88a1e597c28dad65ae6433ecb9f2e4f022f", size = 10886995 }, + { url = "https://files.pythonhosted.org/packages/e9/dc/859d889b4d9356a1a2cdbc1e4a0dda94052bc5b5300098647e51a58c430b/ruff-0.8.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:729850feed82ef2440aa27946ab39c18cb4a8889c1128a6d589ffa028ddcfc22", size = 11220750 }, + { url = "https://files.pythonhosted.org/packages/0b/08/e8f519f61f1d624264bfd6b8829e4c5f31c3c61193bc3cff1f19dbe7626a/ruff-0.8.2-py3-none-win32.whl", hash = "sha256:ac42caaa0411d6a7d9594363294416e0e48fc1279e1b0e948391695db2b3d5b1", size = 8729396 }, + { url = "https://files.pythonhosted.org/packages/f8/d4/ba1c7ab72aba37a2b71fe48ab95b80546dbad7a7f35ea28cf66fc5cea5f6/ruff-0.8.2-py3-none-win_amd64.whl", hash = "sha256:2aae99ec70abf43372612a838d97bfe77d45146254568d94926e8ed5bbb409ea", size = 9594729 }, + { url = "https://files.pythonhosted.org/packages/23/34/db20e12d3db11b8a2a8874258f0f6d96a9a4d631659d54575840557164c8/ruff-0.8.2-py3-none-win_arm64.whl", hash = "sha256:fb88e2a506b70cfbc2de6fae6681c4f944f7dd5f2fe87233a7233d888bad73e8", size = 9035131 }, ] [[package]] @@ -5608,6 +5973,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", size = 78154 }, ] +[[package]] +name = "texttable" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/dc/0aff23d6036a4d3bf4f1d8c8204c5c79c4437e25e0ae94ffe4bbb55ee3c2/texttable-1.7.0.tar.gz", hash = "sha256:2d2068fb55115807d3ac77a4ca68fa48803e84ebb0ee2340f858107a36522638", size = 12831 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl", hash = "sha256:72227d592c82b3d7f672731ae73e4d1f88cd8e2ef5b075a7a7f01a23a3743917", size = 10768 }, +] + [[package]] name = "threadpoolctl" version = "3.5.0" @@ -6125,7 +6499,7 @@ dev = [ { name = "pytest", specifier = ">=6.2.4" }, { name = "pyyaml", specifier = ">=6.0.1" }, { name = "requests-mock", specifier = ">=1.9.3" }, - { name = "ruff", specifier = "==0.1.6" }, + { name = "ruff", specifier = "==0.8.2" }, { name = "tqdm", specifier = ">=4.62.3" }, { name = "types-pyyaml", specifier = ">=6.0.5" }, { name = "types-requests", specifier = ">=2.25.2" }, diff --git a/vscode_extensions/find-latest-etl-step/src/extension.ts b/vscode_extensions/find-latest-etl-step/src/extension.ts index bb1f0f2bdd6..8b3277d7fd7 100644 --- a/vscode_extensions/find-latest-etl-step/src/extension.ts +++ b/vscode_extensions/find-latest-etl-step/src/extension.ts @@ -115,9 +115,7 @@ function findFiles(dir: string, ig: any): { path: string, date: Date | 'latest', const excludeFolders = [ path.join('etl', 'data'), path.join('etl', 'export'), - path.join('snapshots', 'backport'), - 'snapshots_archive', - path.join('etl', 'steps', 'archive') + path.join('snapshots', 'backport') ]; if (excludeFolders.some(excludeFolder => filePath.includes(excludeFolder)) || filePath.includes('__pycache__')) {