From b64b890a7a037923e3d52249391488c1de0b225c Mon Sep 17 00:00:00 2001 From: Jean Date: Sun, 7 Feb 2021 20:34:25 +0100 Subject: [PATCH 1/8] feat: alembic database migration config --- requirements.txt | 1 + src/alembic.ini | 84 ++++++++++++++++++++++++++++++++++++++ src/alembic/README.md | 22 ++++++++++ src/alembic/env.py | 82 +++++++++++++++++++++++++++++++++++++ src/alembic/script.py.mako | 24 +++++++++++ 5 files changed, 213 insertions(+) create mode 100644 src/alembic.ini create mode 100644 src/alembic/README.md create mode 100644 src/alembic/env.py create mode 100644 src/alembic/script.py.mako diff --git a/requirements.txt b/requirements.txt index 25d1c370..4fecba20 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ fastapi==0.61.1 uvicorn>=0.11.1 databases[postgresql]>=0.2.6,<=0.4.0 SQLAlchemy>=1.3.12 +alembic==1.5.4 python-jose>=3.2.0 passlib[bcrypt]>=1.7.4 python-multipart==0.0.5 diff --git a/src/alembic.ini b/src/alembic.ini new file mode 100644 index 00000000..b5d3f470 --- /dev/null +++ b/src/alembic.ini @@ -0,0 +1,84 @@ +# A generic, single database configuration. + +# TODO adapt if needed + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# timezone to use when rendering the date +# within the migration file as well as the filename. +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat alembic/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks=black +# black.type=console_scripts +# black.entrypoint=black +# black.options=-l 79 + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/src/alembic/README.md b/src/alembic/README.md new file mode 100644 index 00000000..8a757ac0 --- /dev/null +++ b/src/alembic/README.md @@ -0,0 +1,22 @@ +[comment]: <> (TODO write migration guide/contributing with docker-compose commands) + +# Alembic main commands + +## Manually created revision + +-> `$ alembic revision -m "create account table"`
+generates template "alembic/{id}_create_account_table.py" + +-> implement `upgrade()` & `downgrade()` functions using SQLAlchemy operations (op.create_table, op.add_column, etc.) + +-> `$ alembic upgrade head` +applies all pending revisions to database + + +## Auto generated revision +**Alternative** using auto generated revision (https://alembic.sqlalchemy.org/en/latest/autogenerate.html) + +-> `$ alembic revision --autogenerate -m "Add account ts column"` +generates filled "alembic/{id}_add_account_ts_column.py" + +-> review and adjust generated script \ No newline at end of file diff --git a/src/alembic/env.py b/src/alembic/env.py new file mode 100644 index 00000000..0b4f7ffb --- /dev/null +++ b/src/alembic/env.py @@ -0,0 +1,82 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +from app import config as cfg +from app.db import metadata + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# for 'autogenerate' support +target_metadata = metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def get_db_url(): + return cfg.DATABASE_URL + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + context.configure( + url=get_db_url(), + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + configuration = config.get_section(config.config_ini_section) + configuration["sqlalchemy.url"] = get_db_url() + connectable = engine_from_config( + configuration, + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/src/alembic/script.py.mako b/src/alembic/script.py.mako new file mode 100644 index 00000000..2c015630 --- /dev/null +++ b/src/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} From 51f6d75d0be12a31a4eca5dfba6951c196328895 Mon Sep 17 00:00:00 2001 From: Jean Date: Sun, 7 Feb 2021 20:35:32 +0100 Subject: [PATCH 2/8] chore: PYTHONPATH app/ to make python app modules importable --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 9b87970f..2cef7701 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,8 @@ WORKDIR /usr/src/app # set environment variables ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 +ENV PYTHONPATH "${PYTHONPATH}:/app" + # copy requirements file COPY ./requirements.txt /usr/src/app/requirements.txt From 55f833e0dd5e5f6b1dcd6185fb47867478775df6 Mon Sep 17 00:00:00 2001 From: Jean Date: Sun, 7 Feb 2021 20:38:15 +0100 Subject: [PATCH 3/8] chore: github workflow placeholder for alembic db migration --- .github/workflows/main.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c7a5a269..29d1545b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -102,3 +102,15 @@ jobs: run: | sleep 5 python scripts/api_e2e.py 8002 + +# TODO add a step to run db migration, needs DATABASE_URL through secrets ? + +# db-migration: +# runs-on: ubuntu-latest +# needs: end-to-end +# steps: +# - name: Run database migration +# env: +# DATABASE_URL: ${{ secrets.DATABASE_URL }} +# run: | +# alembic upgrade head \ No newline at end of file From 7a97ad18c90fcccf4468c187fe3fe64c7b69264c Mon Sep 17 00:00:00 2001 From: Jean Date: Wed, 17 Mar 2021 22:01:17 +0100 Subject: [PATCH 4/8] doc: alembic contributing link --- CONTRIBUTING.md | 5 +++++ src/alembic/README.md | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 34f76c07..1f14fc41 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -60,3 +60,8 @@ To ensure that your incoming PR complies with the lint settings, you need to ins flake8 ./ ``` This will read the `.flake8` setting file and let you know whether your commits need some adjustments. + + +### Database schema migration + +- See [Alembic](src/alembic) guide to create revision and run it locally. \ No newline at end of file diff --git a/src/alembic/README.md b/src/alembic/README.md index 8a757ac0..784f1137 100644 --- a/src/alembic/README.md +++ b/src/alembic/README.md @@ -1,7 +1,7 @@ -[comment]: <> (TODO write migration guide/contributing with docker-compose commands) - # Alembic main commands +Following main guidelines, following commands should be run within a running container using docker-compose: `$ PORT=8002 docker-compose exec -T web alembic ...` + ## Manually created revision -> `$ alembic revision -m "create account table"`
From 9c456edc3ffc776af10345ef867db8e2f8595e7e Mon Sep 17 00:00:00 2001 From: Jean Date: Wed, 17 Mar 2021 22:02:03 +0100 Subject: [PATCH 5/8] chore: move alembic to requirements-dev --- requirements.txt | 1 - src/requirements-dev.txt | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4fecba20..25d1c370 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,6 @@ fastapi==0.61.1 uvicorn>=0.11.1 databases[postgresql]>=0.2.6,<=0.4.0 SQLAlchemy>=1.3.12 -alembic==1.5.4 python-jose>=3.2.0 passlib[bcrypt]>=1.7.4 python-multipart==0.0.5 diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 9dd4bb28..91f89b2f 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -4,3 +4,4 @@ asyncpg>=0.20.0 coverage>=4.5.4 aiosqlite>=0.16.0 httpx>=0.16.1 +alembic==1.5.4 \ No newline at end of file From bca73f7a1f6465faaaf75dcd370b6b9a9fb4fad4 Mon Sep 17 00:00:00 2001 From: Jean Date: Wed, 17 Mar 2021 22:03:48 +0100 Subject: [PATCH 6/8] chore: remove gh workflow comments --- .github/workflows/main.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 29d1545b..c7a5a269 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -102,15 +102,3 @@ jobs: run: | sleep 5 python scripts/api_e2e.py 8002 - -# TODO add a step to run db migration, needs DATABASE_URL through secrets ? - -# db-migration: -# runs-on: ubuntu-latest -# needs: end-to-end -# steps: -# - name: Run database migration -# env: -# DATABASE_URL: ${{ secrets.DATABASE_URL }} -# run: | -# alembic upgrade head \ No newline at end of file From f78ca729020653f852e3f5fe41e008ddf8ae152f Mon Sep 17 00:00:00 2001 From: Jean Date: Wed, 17 Mar 2021 22:05:09 +0100 Subject: [PATCH 7/8] chore: remove useless alembic.ini comments --- src/alembic.ini | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/alembic.ini b/src/alembic.ini index b5d3f470..d976da96 100644 --- a/src/alembic.ini +++ b/src/alembic.ini @@ -1,7 +1,3 @@ -# A generic, single database configuration. - -# TODO adapt if needed - [alembic] # path to migration scripts script_location = alembic From 52717676744599fac504184a428703b6a355f7d5 Mon Sep 17 00:00:00 2001 From: frgfm Date: Tue, 23 Mar 2021 22:52:37 +0100 Subject: [PATCH 8/8] docs: Fixed copyright + license notice --- src/alembic/env.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/alembic/env.py b/src/alembic/env.py index 0b4f7ffb..60c40053 100644 --- a/src/alembic/env.py +++ b/src/alembic/env.py @@ -1,3 +1,8 @@ +# Copyright (C) 2021, Pyronear contributors. + +# This program is licensed under the Apache License version 2. +# See LICENSE or go to for full license details. + from logging.config import fileConfig from sqlalchemy import engine_from_config