Skip to content

Commit

Permalink
Revert ":hammer: refactor metadata classes (#2697)"
Browse files Browse the repository at this point in the history
This reverts commit da278aa.
  • Loading branch information
Marigold committed May 29, 2024
1 parent ecfcff5 commit ab21fd4
Showing 1 changed file with 114 additions and 46 deletions.
160 changes: 114 additions & 46 deletions lib/catalog/owid/catalog/meta.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,11 @@
import re
from dataclasses import dataclass, field, is_dataclass
from pathlib import Path
from typing import Any, Dict, List, Literal, NewType, Optional, Type, TypeVar, Union
from typing import Any, Dict, List, Literal, NewType, Optional, Union

import mistune
import pandas as pd
from dataclasses_json import dataclass_json
from typing_extensions import Self

from .processing_log import ProcessingLog
from .utils import pruned_json
Expand All @@ -27,52 +26,24 @@
YearDateLatest = NewType("YearDateLatest", str)


T = TypeVar("T")

@pruned_json
@dataclass_json
@dataclass
class License:
name: Optional[str] = None
url: Optional[str] = None

class MetaBase:
def __hash__(self):
"""Hash that uniquely identifies an object (without needing frozen dataclass)."""
"""Hash that uniquely identifies a License."""
return _hash_dataclass(self)

def to_dict(self) -> Dict[str, Any]:
...

@classmethod
def from_dict(cls: Type[T], d: Dict[str, Any]) -> T:
@staticmethod
def from_dict(d: Dict[str, Any]) -> "License":
...

def update(self, **kwargs: Dict[str, Any]) -> None:
"""Update object with new values."""
for key, value in kwargs.items():
if value is not None:
setattr(self, key, value)

def copy(self, deep=True) -> Self:
"""Return a copy of the object."""
if not deep:
return dataclasses.replace(self) # type: ignore
else:
return _deepcopy_dataclass(self)

def save(self, filename: Union[str, Path]) -> None:
filename = Path(filename).as_posix()
with open(filename, "w") as ostream:
json.dump(self.to_dict(), ostream, indent=2, default=str)

@classmethod
def load(cls, filename: str) -> Self:
with open(filename) as istream:
return cls.from_dict(json.load(istream))


@pruned_json
@dataclass_json
@dataclass
class License(MetaBase):
name: Optional[str] = None
url: Optional[str] = None

def __bool__(self):
return bool(self.name or self.url)

Expand All @@ -81,7 +52,7 @@ def __bool__(self):
@pruned_json
@dataclass_json
@dataclass
class Source(MetaBase):
class Source:
"""Notes on importing sources to grapher:
- Field `source.description` gets mapped to `Internal notes`, but we rather use it for `additional_info`
- The most important fields are `published_by` and `additional_info`
Expand All @@ -102,11 +73,27 @@ class Source(MetaBase):
# we're keeping both for the time being. We might consolidate them in the future
published_by: Optional[str] = None

def __hash__(self):
"""Hash that uniquely identifies a source."""
return _hash_dataclass(self)

def to_dict(self) -> Dict[str, Any]:
...

@staticmethod
def from_dict(d: Dict[str, Any]) -> "Source":
...

def update(self, **kwargs: Dict[str, Any]) -> None:
for key, value in kwargs.items():
if value is not None:
setattr(self, key, value)


@pruned_json
@dataclass_json
@dataclass
class Origin(MetaBase):
class Origin:
# Producer name
# Name of the institution or the author(s) that produced the data product.
producer: str
Expand Down Expand Up @@ -140,6 +127,10 @@ class Origin(MetaBase):
# License of the dataset
license: Optional[License] = None

def __hash__(self):
"""Hash that uniquely identifies an origin."""
return _hash_dataclass(self)

def __post_init__(self):
if self.date_published:
# convert date or int to string
Expand All @@ -149,6 +140,18 @@ def __post_init__(self):
if self.date_published != "latest" and not is_year_or_date(self.date_published):
raise ValueError("date_published should be either a year or a date or latest")

def to_dict(self) -> Dict[str, Any]:
...

@staticmethod
def from_dict(d: Dict[str, Any]) -> "Origin":
...

def update(self, **kwargs: Dict[str, Any]) -> None:
for key, value in kwargs.items():
if value is not None:
setattr(self, key, value)


# Minor is for cases where we only harmonized the countries or similar
# Major is for cases where we do more, like create new aggregations, combine multiple indicators, etc.
Expand All @@ -164,7 +167,7 @@ def __post_init__(self):
@pruned_json
@dataclass_json
@dataclass
class FaqLink(MetaBase):
class FaqLink:
gdoc_id: str
fragment_id: str

Expand All @@ -175,7 +178,7 @@ class FaqLink(MetaBase):
@pruned_json
@dataclass_json
@dataclass
class VariablePresentationMeta(MetaBase):
class VariablePresentationMeta:
# Any fields of grapher config can be set here - title and subtitle *should* be set whenever possible
grapher_config: Optional[GrapherConfig] = None
# The text for the header of the data page
Expand All @@ -194,11 +197,19 @@ class VariablePresentationMeta(MetaBase):
# List of google doc ids + fragment id
faqs: List[FaqLink] = field(default_factory=list)

def __hash__(self):
"""Hash that uniquely identifies VariablePresentationMeta."""
return _hash_dataclass(self)

@staticmethod
def from_dict(d: Dict[str, Any]) -> "VariablePresentationMeta":
...


@pruned_json
@dataclass_json
@dataclass
class VariableMeta(MetaBase):
class VariableMeta:
"""Allowed fields for `display` attribute used for grapher:
name
zeroDay
Expand Down Expand Up @@ -256,6 +267,17 @@ class VariableMeta(MetaBase):
# List of categories for ordinal type indicators
sort: List[str] = field(default_factory=list)

def __hash__(self):
"""Hash that uniquely identifies VariableMeta."""
return _hash_dataclass(self)

def to_dict(self) -> Dict[str, Any]:
...

@staticmethod
def from_dict(d: Dict[str, Any]) -> "VariableMeta":
...

@property
def schema_version(self) -> int:
"""Schema version is used to easily understand everywhere what metadata standard was used
Expand All @@ -276,11 +298,18 @@ def _repr_html_(self):
{}
""".format(getattr(self, "_name", None), to_html(record))

def copy(self, deep=True) -> "VariableMeta":
"""Return a copy of the VariableMeta object."""
if not deep:
return dataclasses.replace(self)
else:
return _deepcopy_dataclass(self)


@pruned_json
@dataclass_json
@dataclass
class DatasetMeta(MetaBase):
class DatasetMeta:
"""
The metadata for this entire dataset kept in JSON (e.g. mydataset/index.json).
Expand Down Expand Up @@ -310,6 +339,10 @@ class DatasetMeta(MetaBase):
# an md5 checksum of the ingredients used to make this dataset
source_checksum: Optional[str] = None

def __hash__(self):
"""Hash that uniquely identifies DatasetMeta."""
return _hash_dataclass(self)

def __post_init__(self) -> None:
"""Imply version from publication_date or publication_year if not given
in __init__."""
Expand All @@ -323,6 +356,23 @@ def __post_init__(self) -> None:
else:
self.version = None

def save(self, filename: Union[str, Path]) -> None:
filename = Path(filename).as_posix()
with open(filename, "w") as ostream:
json.dump(self.to_dict(), ostream, indent=2, default=str)

@classmethod
def load(cls, filename: str) -> "DatasetMeta":
with open(filename) as istream:
return cls.from_dict(json.load(istream))

def to_dict(self) -> Dict[str, Any]:
...

@staticmethod
def from_dict(d: Dict[str, Any]) -> "DatasetMeta":
...

def _params_yaml(self) -> dict:
"""Parameters passed to YAML for dynamic interpolation."""
params = {}
Expand Down Expand Up @@ -382,7 +432,7 @@ def uri(self) -> str:
@pruned_json
@dataclass_json
@dataclass
class TableMeta(MetaBase):
class TableMeta:
# data about this table
short_name: Optional[str] = None
title: Optional[str] = None
Expand All @@ -392,13 +442,24 @@ class TableMeta(MetaBase):
dataset: Optional[DatasetMeta] = field(compare=False, default=None)
primary_key: List[str] = field(default_factory=list)

def __hash__(self):
"""Hash that uniquely identifies TableMeta."""
return _hash_dataclass(self)

@property
def checked_name(self) -> str:
if not self.short_name:
raise Exception("table has no short_name")

return self.short_name

def to_dict(self) -> Dict[str, Any]:
...

@staticmethod
def from_dict(dict: Dict[str, Any]) -> "TableMeta":
...

def _repr_html_(self):
# Render a nice display of the table metadata
record = self.to_dict()
Expand All @@ -409,6 +470,13 @@ def _repr_html_(self):
{}
""".format(short_name, to_html(record))

def copy(self, deep=True) -> "TableMeta":
"""Return a copy of the TableMeta object."""
if not deep:
return dataclasses.replace(self)
else:
return _deepcopy_dataclass(self)


def to_html(record: Any) -> Optional[str]:
if isinstance(record, dict):
Expand Down

0 comments on commit ab21fd4

Please sign in to comment.