From 213385161cae08e49cb1d7e4293698640dfbfcee Mon Sep 17 00:00:00 2001 From: Marigold Date: Wed, 22 May 2024 13:59:49 +0200 Subject: [PATCH] move to dataclass based ORM classes --- apps/backport/backport.py | 1 + etl/grapher_model.py | 195 ++++++++++++++++--------------- lib/catalog/owid/catalog/meta.py | 2 +- tests/test_grapher_model.py | 2 +- 4 files changed, 103 insertions(+), 97 deletions(-) diff --git a/apps/backport/backport.py b/apps/backport/backport.py index 4ddcc3f581c8..e1e141640900 100644 --- a/apps/backport/backport.py +++ b/apps/backport/backport.py @@ -267,6 +267,7 @@ def _upload_data_metadata(lg: Any, backport_short_name: str, dry_run: bool) -> N # artificial variable with id just to get s3 paths db_var = gm.Variable( id=db_variable_row["id"], + description="", datasetId=1, unit="", coverage="", diff --git a/etl/grapher_model.py b/etl/grapher_model.py index ad1af6f79577..7ccb60e3737c 100644 --- a/etl/grapher_model.py +++ b/etl/grapher_model.py @@ -5,6 +5,8 @@ If you want to add a new table to ORM, add --tables mytable to the command above. Another option is to run `show create table mytable;` in MySQL and then ask ChatGPT to convert it to SQLAlchemy 2 ORM. + +It is often necessary to add `default=None` or `init=False` to make pyright happy. """ import json from datetime import date, datetime @@ -42,7 +44,7 @@ VARCHAR, ) from sqlalchemy.exc import NoResultFound -from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column +from sqlalchemy.orm import DeclarativeBase, Mapped, MappedAsDataclass, Session, mapped_column from sqlalchemy.sql import Select from typing_extensions import TypedDict @@ -59,8 +61,7 @@ JSON = _JSON(none_as_null=True) -# class Base(MappedAsDataclass, DeclarativeBase): -class Base(DeclarativeBase): +class Base(MappedAsDataclass, DeclarativeBase): __table_args__ = {"extend_existing": True} def dict(self) -> Dict[str, Any]: @@ -96,25 +97,25 @@ class Entity(Base): __tablename__ = "entities" __table_args__ = (Index("code", "code", unique=True), Index("name", "name", unique=True)) - id: Mapped[int] = mapped_column(Integer, primary_key=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True, init=False) name: Mapped[str] = mapped_column(VARCHAR(255)) validated: Mapped[int] = mapped_column(TINYINT(1)) - createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP")) + createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP"), init=False) displayName: Mapped[str] = mapped_column(VARCHAR(255)) code: Mapped[Optional[str]] = mapped_column(VARCHAR(255)) - updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime) + updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime, init=False) class Namespace(Base): __tablename__ = "namespaces" __table_args__ = (Index("namespaces_name_uq", "name", unique=True),) - id: Mapped[int] = mapped_column(Integer, primary_key=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True, init=False) name: Mapped[str] = mapped_column(VARCHAR(255)) - isArchived: Mapped[int] = mapped_column(TINYINT(1), server_default=text("'0'")) - createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP")) - description: Mapped[Optional[str]] = mapped_column(VARCHAR(255)) - updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime) + isArchived: Mapped[int] = mapped_column(TINYINT(1), server_default=text("'0'"), default=0) + createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP"), init=False) + description: Mapped[Optional[str]] = mapped_column(VARCHAR(255), default=None) + updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime, init=False) def upsert(self, session: Session) -> "Namespace": cls = self.__class__ @@ -146,22 +147,22 @@ class Tag(Base): Index("slug", "slug", unique=True), ) - id: Mapped[int] = mapped_column(Integer, primary_key=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True, init=False) name: Mapped[str] = mapped_column(VARCHAR(255)) - createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP")) - updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime) + createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP"), init=False) + updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime, init=False) parentId: Mapped[Optional[int]] = mapped_column(Integer) specialType: Mapped[Optional[str]] = mapped_column(VARCHAR(255)) slug: Mapped[Optional[str]] = mapped_column(VARCHAR(512)) @classmethod - def load_tags(cls, session: Session) -> List["Tag"]: # type: ignore - return session.scalars(select(cls).where(cls.slug.isnot(None))).all() # type: ignore + def load_tags(cls, session: Session) -> List["Tag"]: + return list(session.scalars(select(cls).where(cls.slug.isnot(None))).all()) @classmethod def load_tags_by_names(cls, session: Session, tag_names: List[str]) -> List["Tag"]: """Load topic tags by their names in the order given in `tag_names`.""" - tags = session.scalars(select(Tag).where(Tag.name.in_(tag_names), Tag.slug.isnot(None))).all() # type: ignore + tags = session.scalars(select(Tag).where(Tag.name.in_(tag_names), Tag.slug.isnot(None))).all() if len(tags) != len(tag_names): found_tags = [tag.name for tag in tags] @@ -177,15 +178,15 @@ class User(Base): __tablename__ = "users" __table_args__ = (Index("email", "email", unique=True),) - id: Mapped[int] = mapped_column(Integer, primary_key=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True, init=False) isSuperuser: Mapped[int] = mapped_column(TINYINT(1), server_default=text("'0'")) email: Mapped[str] = mapped_column(VARCHAR(255)) - createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP")) + createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP"), init=False) isActive: Mapped[int] = mapped_column(TINYINT(1), server_default=text("'1'")) fullName: Mapped[str] = mapped_column(VARCHAR(255)) password: Mapped[Optional[str]] = mapped_column(VARCHAR(128)) lastLogin: Mapped[Optional[datetime]] = mapped_column(DateTime) - updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime) + updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime, init=False) lastSeen: Mapped[Optional[datetime]] = mapped_column(DateTime) @@ -200,11 +201,11 @@ class ChartRevisions(Base): ) id: Mapped[int] = mapped_column(BigInteger, primary_key=True) - createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP")) + createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP"), init=False) chartId: Mapped[Optional[int]] = mapped_column(Integer) userId: Mapped[Optional[int]] = mapped_column(Integer) config: Mapped[Optional[dict]] = mapped_column(JSON) - updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime) + updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime, init=False) class Chart(Base): @@ -229,9 +230,9 @@ class Chart(Base): Index("charts_slug", "slug"), ) - id: Mapped[int] = mapped_column(Integer, primary_key=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True, init=False) config: Mapped[dict] = mapped_column(JSON) - createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP")) + createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP"), init=False) lastEditedAt: Mapped[datetime] = mapped_column(DateTime) lastEditedByUserId: Mapped[int] = mapped_column(Integer) is_indexable: Mapped[int] = mapped_column(TINYINT(1), server_default=text("'0'")) @@ -244,7 +245,7 @@ class Chart(Base): "(coalesce(json_unquote(json_extract(`config`,_utf8mb4'$.type')),_utf8mb4'LineChart'))", persisted=False ), ) - updatedAt: Mapped[datetime] = mapped_column(DateTime) + updatedAt: Mapped[datetime] = mapped_column(DateTime, init=False) publishedAt: Mapped[Optional[datetime]] = mapped_column(DateTime) publishedByUserId: Mapped[Optional[int]] = mapped_column(Integer) @@ -272,22 +273,24 @@ def load_charts_using_variables(cls, session: Session, variable_ids: List[int]) """Load charts that use any of the given variables in `variable_ids`.""" # Find IDs of charts chart_ids = ( - session.scalars(select(ChartDimensions.chartId).where(ChartDimensions.variableId.in_(variable_ids))) # type: ignore + session.scalars(select(ChartDimensions.chartId).where(ChartDimensions.variableId.in_(variable_ids))) .unique() .all() ) # Find charts - return session.scalars(select(Chart).where(Chart.id.in_(chart_ids))).all() # type: ignore + return list(session.scalars(select(Chart).where(Chart.id.in_(chart_ids))).all()) def load_chart_variables(self, session: Session) -> Dict[int, "Variable"]: - q = """ + q = text( + """ select v.* from chart_dimensions as cd join variables as v on v.id = cd.variableId where cd.chartId = :chart_id """ - rows = session.scalars(q, params={"chart_id": self.id}).fetchall() # type: ignore + ) + rows = session.scalars(q, params={"chart_id": self.id}).fetchall() variables = {r["id"]: Variable(**r) for r in rows} # add columnSlug if present @@ -347,7 +350,8 @@ def migrate_to_db(self, source_session: Session, target_session: Session) -> "Ch def tags(self, session: Session) -> List[Dict[str, Any]]: """Return tags in a format suitable for Admin API.""" - q = """ + q = text( + """ select tagId as id, t.name, @@ -357,8 +361,9 @@ def tags(self, session: Session) -> List[Dict[str, Any]]: join tags as t on ct.tagId = t.id where ct.chartId = :chart_id """ - rows = session.scalars(q, params={"chart_id": self.id}).fetchall() # type: ignore - return rows + ) + rows = session.scalars(q, params={"chart_id": self.id}).fetchall() + return list(rows) def remove_nonexisting_map_column_slug(self, session: Session) -> None: """Remove map.columnSlug if the variable doesn't exist. It'd be better @@ -423,24 +428,24 @@ class Dataset(Base): Index("unique_short_name_version_namespace", "shortName", "version", "namespace", unique=True), ) - id: Mapped[int] = mapped_column(Integer, primary_key=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True, init=False) name: Mapped[str] = mapped_column(VARCHAR(512)) description: Mapped[str] = mapped_column(LONGTEXT) - createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP")) + createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP"), init=False) namespace: Mapped[str] = mapped_column(VARCHAR(255)) isPrivate: Mapped[int] = mapped_column(TINYINT(1), server_default=text("'0'")) createdByUserId: Mapped[int] = mapped_column(Integer) - metadataEditedAt: Mapped[datetime] = mapped_column(DateTime) + metadataEditedAt: Mapped[datetime] = mapped_column(DateTime, init=False) metadataEditedByUserId: Mapped[int] = mapped_column(Integer) - dataEditedAt: Mapped[datetime] = mapped_column(DateTime) + dataEditedAt: Mapped[datetime] = mapped_column(DateTime, init=False) dataEditedByUserId: Mapped[int] = mapped_column(Integer) nonRedistributable: Mapped[int] = mapped_column(TINYINT(1), server_default=text("'0'")) - isArchived: Mapped[int] = mapped_column(TINYINT(1), server_default=text("'0'")) - updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime) - sourceChecksum: Mapped[Optional[str]] = mapped_column(VARCHAR(64)) + updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime, init=False) shortName: Mapped[Optional[str]] = mapped_column(VARCHAR(255)) version: Mapped[Optional[str]] = mapped_column(VARCHAR(255)) updatePeriodDays: Mapped[Optional[int]] = mapped_column(Integer) + isArchived: Mapped[Optional[int]] = mapped_column(TINYINT(1), server_default=text("'0'"), default=0) + sourceChecksum: Mapped[Optional[str]] = mapped_column(VARCHAR(64), default=None) def upsert(self, session: Session) -> "Dataset": cls = self.__class__ @@ -546,12 +551,12 @@ class Source(Base): Index("sources_datasetId", "datasetId"), ) - id: Mapped[int] = mapped_column(Integer, primary_key=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True, init=False) description: Mapped[SourceDescription] = mapped_column(JSON) - createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP")) - name: Mapped[Optional[str]] = mapped_column(VARCHAR(512)) - updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime) - datasetId: Mapped[Optional[int]] = mapped_column(Integer) + createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP"), init=False) + name: Mapped[Optional[str]] = mapped_column(VARCHAR(512), default=None) + updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime, init=False) + datasetId: Mapped[Optional[int]] = mapped_column(Integer, default=None) @property def _upsert_select(self) -> Select: @@ -564,7 +569,7 @@ def _upsert_select(self) -> Select: _json_is(cls.description, "additionalInfo", self.description.get("additionalInfo")), _json_is(cls.description, "dataPublishedBy", self.description.get("dataPublishedBy")), ] - return select(cls).where(*conds) # type: ignore + return select(cls).where(*conds) def upsert(self, session: Session) -> "Source": ds = session.scalars(self._upsert_select).one_or_none() @@ -624,7 +629,7 @@ def load_sources( """ sources = read_sql( q, - session.bind, # type: ignore + session, params={ "datasetId": dataset_id, # NOTE: query doesn't work with empty list so we use a dummy value @@ -642,7 +647,7 @@ def load_sources( ) sources.datasetId = sources.datasetId.fillna(dataset_id).astype(int) - return [cls(**d) for d in sources.to_dict(orient="records") if cls.validate(d)] # type: ignore + return [cls(**d) for d in sources.to_dict(orient="records")] # type: ignore class SuggestedChartRevisions(Base): @@ -740,13 +745,13 @@ class PostsGdocs(Base): slug: Mapped[str] = mapped_column(VARCHAR(255)) content: Mapped[dict] = mapped_column(JSON) published: Mapped[int] = mapped_column(TINYINT) - createdAt: Mapped[datetime] = mapped_column(DateTime) + createdAt: Mapped[datetime] = mapped_column(DateTime, init=False) publicationContext: Mapped[str] = mapped_column(ENUM("unlisted", "listed"), server_default=text("'unlisted'")) type: Mapped[Optional[str]] = mapped_column( VARCHAR(255), Computed("(json_unquote(json_extract(`content`,_utf8mb4'$.type')))", persisted=False) ) publishedAt: Mapped[Optional[datetime]] = mapped_column(DateTime) - updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime) + updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime, init=False) revisionId: Mapped[Optional[str]] = mapped_column(VARCHAR(255)) breadcrumbs: Mapped[Optional[dict]] = mapped_column(JSON) markdown: Mapped[Optional[str]] = mapped_column(LONGTEXT) @@ -859,7 +864,7 @@ class TagsVariablesTopicTagsLink(Base): # variable: Mapped["Variable"] = relationship("Variable", back_populates="tags_variables_topic_tagss") @classmethod - def link_with_variable(cls, session: Session, variable_id: int, new_tag_ids: List[str]) -> None: + def link_with_variable(cls, session: Session, variable_id: int, new_tag_ids: List[int]) -> None: """Link the given Variable ID with the given Tag IDs.""" assert len(new_tag_ids) == len(set(new_tag_ids)), "Tag IDs must be unique" @@ -938,42 +943,42 @@ class Variable(Base): ) id: Mapped[int] = mapped_column(Integer, primary_key=True) + description: Mapped[Optional[str]] = mapped_column(LONGTEXT) unit: Mapped[str] = mapped_column(VARCHAR(255)) - createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP")) + createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP"), init=False) coverage: Mapped[str] = mapped_column(VARCHAR(255)) timespan: Mapped[str] = mapped_column(VARCHAR(255)) datasetId: Mapped[int] = mapped_column(Integer) display: Mapped[dict] = mapped_column(JSON) - columnOrder: Mapped[int] = mapped_column(Integer, server_default=text("'0'")) - schemaVersion: Mapped[int] = mapped_column(Integer, server_default=text("'1'")) - name: Mapped[Optional[str]] = mapped_column(VARCHAR(750)) - description: Mapped[Optional[str]] = mapped_column(LONGTEXT) - updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime) - code: Mapped[Optional[str]] = mapped_column(VARCHAR(255)) - sourceId: Mapped[Optional[int]] = mapped_column(Integer) - shortUnit: Mapped[Optional[str]] = mapped_column(VARCHAR(255)) - originalMetadata: Mapped[Optional[dict]] = mapped_column(JSON) - grapherConfigAdmin: Mapped[Optional[dict]] = mapped_column(JSON) - shortName: Mapped[Optional[str]] = mapped_column(VARCHAR(255)) - catalogPath: Mapped[Optional[str]] = mapped_column(VARCHAR(767)) - dimensions: Mapped[Optional[dict]] = mapped_column(JSON) - processingLevel: Mapped[Optional[catalog.meta.PROCESSING_LEVELS]] = mapped_column(VARCHAR(30)) - processingLog: Mapped[Optional[dict]] = mapped_column(JSON) - titlePublic: Mapped[Optional[str]] = mapped_column(VARCHAR(512)) - titleVariant: Mapped[Optional[str]] = mapped_column(VARCHAR(255)) - attributionShort: Mapped[Optional[str]] = mapped_column(VARCHAR(512)) - attribution: Mapped[Optional[str]] = mapped_column(TEXT) - descriptionShort: Mapped[Optional[str]] = mapped_column(TEXT) - descriptionFromProducer: Mapped[Optional[str]] = mapped_column(TEXT) - descriptionKey: Mapped[Optional[dict]] = mapped_column(JSON) - descriptionProcessing: Mapped[Optional[str]] = mapped_column(TEXT) + columnOrder: Mapped[int] = mapped_column(Integer, server_default=text("'0'"), default=0) + schemaVersion: Mapped[int] = mapped_column(Integer, server_default=text("'1'"), default=1) + name: Mapped[Optional[str]] = mapped_column(VARCHAR(750), default=None) + updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime, init=False) + code: Mapped[Optional[str]] = mapped_column(VARCHAR(255), default=None) + sourceId: Mapped[Optional[int]] = mapped_column(Integer, default=None) + shortUnit: Mapped[Optional[str]] = mapped_column(VARCHAR(255), default=None) + originalMetadata: Mapped[Optional[dict]] = mapped_column(JSON, default=None) + grapherConfigAdmin: Mapped[Optional[dict]] = mapped_column(JSON, default=None) + shortName: Mapped[Optional[str]] = mapped_column(VARCHAR(255), default=None) + catalogPath: Mapped[Optional[str]] = mapped_column(VARCHAR(767), default=None) + dimensions: Mapped[Optional[Dimensions]] = mapped_column(JSON, default=None) + processingLevel: Mapped[Optional[catalog.meta.PROCESSING_LEVELS]] = mapped_column(VARCHAR(30), default=None) + processingLog: Mapped[Optional[dict]] = mapped_column(JSON, default=None) + titlePublic: Mapped[Optional[str]] = mapped_column(VARCHAR(512), default=None) + titleVariant: Mapped[Optional[str]] = mapped_column(VARCHAR(255), default=None) + attributionShort: Mapped[Optional[str]] = mapped_column(VARCHAR(512), default=None) + attribution: Mapped[Optional[str]] = mapped_column(TEXT, default=None) + descriptionShort: Mapped[Optional[str]] = mapped_column(TEXT, default=None) + descriptionFromProducer: Mapped[Optional[str]] = mapped_column(TEXT, default=None) + descriptionKey: Mapped[Optional[list[str]]] = mapped_column(JSON, default=None) + descriptionProcessing: Mapped[Optional[str]] = mapped_column(TEXT, default=None) # NOTE: Use of `licenses` is discouraged, they should be captured in origins. - licenses: Mapped[Optional[dict]] = mapped_column(JSON) + licenses: Mapped[Optional[list[dict]]] = mapped_column(JSON, default=None) # NOTE: License should be the resulting license, given all licenses of the indicator’s origins and given the indicator’s processing level. - license: Mapped[Optional[dict]] = mapped_column(JSON) - grapherConfigETL: Mapped[Optional[dict]] = mapped_column(JSON) - type: Mapped[Optional[VARIABLE_TYPE]] = mapped_column(ENUM(*get_args(VARIABLE_TYPE))) - sort: Mapped[Optional[dict]] = mapped_column(JSON) + license: Mapped[Optional[dict]] = mapped_column(JSON, default=None) + grapherConfigETL: Mapped[Optional[dict]] = mapped_column(JSON, default=None) + type: Mapped[Optional[VARIABLE_TYPE]] = mapped_column(ENUM(*get_args(VARIABLE_TYPE)), default=None) + sort: Mapped[Optional[list[str]]] = mapped_column(JSON, default=None) # dataset: Mapped["Dataset"] = relationship("Dataset", back_populates="variabless") # source: Mapped["Source"] = relationship("Source", back_populates="variabless") @@ -995,11 +1000,11 @@ def upsert(self, session: Session) -> "Variable": # try matching on shortName first q = select(cls).where( or_( - cls.shortName == self.shortName, # type: ignore + cls.shortName == self.shortName, # NOTE: we used to slugify shortName which replaced double underscore by a single underscore # this was a bug, we should have kept the double underscore # match even those variables and correct their shortName - cls.shortName == self.shortName.replace("__", "_"), # type: ignore + cls.shortName == self.shortName.replace("__", "_"), ), cls.datasetId == self.datasetId, ) @@ -1098,7 +1103,7 @@ def from_variable_metadata( assert "#" in catalog_path, "catalog_path should end with #indicator_short_name" if metadata.presentation: - presentation_dict = metadata.presentation.to_dict() # type: ignore + presentation_dict = metadata.presentation.to_dict() # convert all fields from snake_case to camelCase presentation_dict = humps.camelize(presentation_dict) else: @@ -1165,11 +1170,11 @@ def update_links( assert self.id # establish relationships between variables and origins - OriginsVariablesLink.link_with_variable(session, self.id, [origin.id for origin in db_origins]) # type: ignore + OriginsVariablesLink.link_with_variable(session, self.id, [origin.id for origin in db_origins]) # establish relationships between variables and posts required_gdoc_ids = {faq.gdoc_id for faq in faqs} - query = select(PostsGdocs).where(PostsGdocs.id.in_(required_gdoc_ids)) # type: ignore + query = select(PostsGdocs).where(PostsGdocs.id.in_(required_gdoc_ids)) gdoc_posts = session.scalars(query).all() existing_gdoc_ids = {gdoc_post.id for gdoc_post in gdoc_posts} missing_gdoc_ids = required_gdoc_ids - existing_gdoc_ids @@ -1182,7 +1187,7 @@ def update_links( # establish relationships between variables and tags tags = Tag.load_tags_by_names(session, tag_names) - TagsVariablesTopicTagsLink.link_with_variable(session, self.id, [tag.id for tag in tags]) # type: ignore + TagsVariablesTopicTagsLink.link_with_variable(session, self.id, [tag.id for tag in tags]) def s3_data_path(self, typ: S3_PATH_TYP = "s3") -> str: """Path to S3 with data in JSON format for Grapher. Typically @@ -1245,13 +1250,13 @@ class ChartDimensions(Base): Index("chart_dimensions_variableId_9ba778e6_fk_variables_id", "variableId"), ) - id: Mapped[int] = mapped_column(Integer, primary_key=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True, init=False) order: Mapped[int] = mapped_column(Integer) property: Mapped[str] = mapped_column(VARCHAR(255)) chartId: Mapped[int] = mapped_column(Integer) variableId: Mapped[int] = mapped_column(Integer) - createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP")) - updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime) + createdAt: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP"), init=False) + updatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime, init=False) # chart: Mapped["Chart"] = relationship("Chart", back_populates="chart_dimensionss") # variable: Mapped["Variable"] = relationship("Variable", back_populates="chart_dimensionss") @@ -1268,7 +1273,7 @@ class Origin(Base): __tablename__ = "origins" - id: Mapped[int] = mapped_column(Integer, primary_key=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True, init=False) titleSnapshot: Mapped[Optional[str]] = mapped_column(VARCHAR(512)) title: Mapped[Optional[str]] = mapped_column(VARCHAR(512)) descriptionSnapshot: Mapped[Optional[str]] = mapped_column(TEXT) @@ -1331,7 +1336,7 @@ def _upsert_select(self) -> Select: cls.description == self.description, cls.datePublished == self.datePublished, cls.dateAccessed == self.dateAccessed, - ) # type: ignore + ) def upsert(self, session: Session) -> "Origin": """ @@ -1380,12 +1385,12 @@ class ChartDiffApprovals(Base): Index("chartId", "chartId"), ) - id: Mapped[int] = mapped_column(Integer, primary_key=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True, init=False) chartId: Mapped[int] = mapped_column(Integer) sourceUpdatedAt: Mapped[datetime] = mapped_column(DateTime) - updatedAt: Mapped[datetime] = mapped_column(DateTime) + updatedAt: Mapped[datetime] = mapped_column(DateTime, init=False) status: Mapped[CHART_DIFF_STATUS] = mapped_column(VARCHAR(255)) - targetUpdatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime) + targetUpdatedAt: Mapped[Optional[datetime]] = mapped_column(DateTime, init=False) # chart: Mapped["Chart"] = relationship("Chart", back_populates="chart_diff_approvalss") @@ -1401,11 +1406,11 @@ def latest_chart_status( cls.sourceUpdatedAt == source_updated_at, cls.targetUpdatedAt == target_updated_at, ) - .order_by(cls.updatedAt.desc()) # type: ignore + .order_by(cls.updatedAt.desc()) .limit(1) ).first() if result: - return result.status # type: ignore + return result.status else: return "unapproved" diff --git a/lib/catalog/owid/catalog/meta.py b/lib/catalog/owid/catalog/meta.py index 0c2a7fb10528..3cebfed2db40 100644 --- a/lib/catalog/owid/catalog/meta.py +++ b/lib/catalog/owid/catalog/meta.py @@ -332,7 +332,7 @@ class DatasetMeta: additional_info: Optional[Dict[str, Any]] = None version: Optional[str] = None # update period in days - update_period_days: Optional[str] = None + update_period_days: Optional[int] = None # prohibit redistribution (disable chart download) non_redistributable: bool = False diff --git a/tests/test_grapher_model.py b/tests/test_grapher_model.py index c03846053f69..e0017f697ae5 100644 --- a/tests/test_grapher_model.py +++ b/tests/test_grapher_model.py @@ -5,6 +5,6 @@ def test_source_description(): """Make sure description as a TypedDict works correctly""" description: gm.SourceDescription = {"link": "ABC"} d = {"description": description} - s = gm.Source(**d) + s = gm.Source(**d) # type: ignore assert "link" in s.description assert s.description["link"] == "ABC"