Add explicit migrations, fake option to update applied migrations, dr…

…op 3.8 python Resolve #27 (#28)

.github/workflows/ci.yaml

@@ -9,7 +9,7 @@ on:
     paths:
       - .github/workflows/**
       - src/**
-      - setup.cfg
+      - dev/**
       - tox.ini
 
 jobs:
@@ -29,7 +29,6 @@ jobs:
           - "3.11"
           - "3.10"
           - "3.9"
-          - "3.8"
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4

CHANGELOG.md

@@ -0,0 +1,29 @@
+# Changelog
+
+## [v0.8.0](https://github.com/zifter/clickhouse-migrations/tree/v0.8.0) (2024-08-18)
+
+[Full Changelog](https://github.com/zifter/clickhouse-migrations/compare/v0.7.1...v0.8.0)
+
+**What's Changed:**
+- Add option --fake/--no-fake, which can help update schema_version without executing statements from migration files #27. Done by @zifter in https://github.com/zifter/clickhouse-migrations/pull/28
+- Add option --migrations which can help to specify explicitly migrations to apply. Done by @zifter in https://github.com/zifter/clickhouse-migrations/pull/28
+
+**Breaking changes:**
+- Drop python 3.8 support. Done by @zifter in https://github.com/zifter/clickhouse-migrations/pull/28
+- Option --multi-statement, --dry-run, --secure now working without passing value. Just use --multi-statement/--no-multi-statement, --dry-run/--no-dry-run, --secure/--no-secure for enabling or disabling option. Done by @zifter in https://github.com/zifter/clickhouse-migrations/pull/28
+
+
+## [v0.7.1](https://github.com/zifter/clickhouse-migrations/tree/v0.7.1) (2024-07-01)
+
+[Full Changelog](https://github.com/zifter/clickhouse-migrations/compare/v0.7.0...v0.7.1)
+
+**What's Changed:**
+- Allow default db name #24. Done by @zifter in https://github.com/zifter/clickhouse-migrations/pull/26
+
+
+## [v0.7.0](https://github.com/zifter/clickhouse-migrations/tree/v0.7.0) (2024-07-01)
+
+[Full Changelog](https://github.com/zifter/clickhouse-migrations/compare/v0.6.0...v0.7.0)
+
+**What's Changed:**
+- #24 Allow connection string for initialization of ClickhouseCluster. Done by @zifter in https://github.com/zifter/clickhouse-migrations/pull/25

README.md

@@ -50,17 +50,20 @@ cluster = ClickhouseCluster(db_host, db_user, db_password)
 cluster.migrate(db_name, migrations_home, cluster_name=None,create_db_if_no_exists=True, multi_statement=True)
 ```
 
-Parameter | Description                                                       | Default
--------|-------------------------------------------------------------------|---------
-db_host | Clickhouse database hostname                                      | localhost
-db_port | Clickhouse database port                                          | 9000
-db_user | Clickhouse user                                                   | default
-db_password | Clichouse password                                                | default
-db_name| Clickhouse database name                                          | None
-migrations_home | Path to list of migration files                                   | <project_root>
-cluster_name | Name of Clickhouse topology cluster from <remote_servers>            | None
-create_db_if_no_exists | If the `db_name` is not present, enabling this will create the db | True
-multi_statement | Allow multiple statements in migration files                      | True
+Parameter | Description                                                                                         | Default
+-------|-----------------------------------------------------------------------------------------------------|---------
+db_host | Clickhouse database hostname                                                                        | localhost
+db_port | Clickhouse database port                                                                            | 9000
+db_user | Clickhouse user                                                                                     | default
+db_password | Clichouse password                                                                                  | default
+db_name| Clickhouse database name                                                                            | None
+migration_path | Path to list of migration files                                                                     | <project_root>
+migrations | Explicit list of migrations to apply                                                                | []
+cluster_name | Name of Clickhouse topology cluster from <remote_servers>                                           | None
+create_db_if_no_exists | If the `db_name` is not present, enabling this will create the db                                   | True
+multi_statement | Allow multiple statements in migration files                                                        | True
+secure | Use secure connection                                                                               | False
+fake | Marks the migrations as applied but without actually running the SQL to change your database schema | False
 
 ### Notes
 The Clickhouse driver does not natively support executing multipe statements in a single query.

pyproject.toml

@@ -8,7 +8,9 @@ authors = [
 urls.Homepage = "https://github.com/zifter/clickhouse-migrations"
 urls.Source = "https://github.com/zifter/clickhouse-migrations"
 urls.Tracker = "https://github.com/zifter/clickhouse-migrations/issues"
-requires-python = ">=3.7, <4"
+urls.Changelog = "https://github.com/zifter/clickhouse-migrations/blob/main/CHANGELOG.md"
+
+requires-python = ">=3.9, <4"
 keywords = [
     "clickhouse",
     "migrations",
@@ -17,7 +19,6 @@ license = {text = "MIT"}
 classifiers = [
     "Intended Audience :: Developers",
     "Programming Language :: Python",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",

src/clickhouse_migrations/clickhouse_cluster.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import List, Optional
+from typing import List, Optional, Union
 
 from clickhouse_driver import Client
 
@@ -87,16 +87,18 @@ def show_tables(self, db_name):
     def migrate(
         self,
         db_name: Optional[str],
-        migration_path: Path,
+        migration_path: Union[Path, str],
         cluster_name: Optional[str] = None,
         create_db_if_no_exists: bool = True,
         multi_statement: bool = True,
         dryrun: bool = False,
+        explicit_migrations: Optional[List[str]] = None,
+        fake: bool = False,
     ):
         db_name = db_name if db_name is not None else self.default_db_name
 
         storage = MigrationStorage(migration_path)
-        migrations = storage.migrations()
+        migrations = storage.migrations(explicit_migrations)
 
         return self.apply_migrations(
             db_name,
@@ -105,6 +107,7 @@ def migrate(
             create_db_if_no_exists=create_db_if_no_exists,
             multi_statement=multi_statement,
             dryrun=dryrun,
+            fake=fake,
         )
 
     def apply_migrations(
@@ -115,6 +118,7 @@ def apply_migrations(
         cluster_name: Optional[str] = None,
         create_db_if_no_exists: bool = True,
         multi_statement: bool = True,
+        fake: bool = False,
     ) -> List[Migration]:
         if create_db_if_no_exists:
             if cluster_name is None:
@@ -125,4 +129,4 @@ def apply_migrations(
         with self.connection(db_name) as conn:
             migrator = Migrator(conn, dryrun)
             migrator.init_schema(cluster_name)
-            return migrator.apply_migration(migrations, multi_statement)
+            return migrator.apply_migration(migrations, multi_statement, fake=fake)

src/clickhouse_migrations/command_line.py

@@ -1,8 +1,10 @@
+import argparse
 import logging
 import os
 import sys
 from argparse import ArgumentParser
 from pathlib import Path
+from typing import List
 
 from clickhouse_migrations.clickhouse_cluster import ClickhouseCluster
 from clickhouse_migrations.defaults import (
@@ -12,6 +14,8 @@
     DB_USER,
     MIGRATIONS_DIR,
 )
+from clickhouse_migrations.migration import Migration
+from clickhouse_migrations.migrator import Migrator
 
 
 def log_level(value: str) -> str:
@@ -35,6 +39,7 @@ def get_context(args):
     parser = ArgumentParser()
     parser.register("type", bool, cast_to_bool)
 
+    default_migrations = os.environ.get("MIGRATIONS", "")
     # detect configuration
     parser.add_argument(
         "--db-url",
@@ -74,8 +79,9 @@ def get_context(args):
     )
     parser.add_argument(
         "--multi-statement",
-        default=os.environ.get("MULTI_STATEMENT", "1"),
+        default=cast_to_bool(os.environ.get("MULTI_STATEMENT", "1")),
         type=bool,
+        action=argparse.BooleanOptionalAction,
         help="Path to list of migration files",
     )
     parser.add_argument(
@@ -91,40 +97,75 @@ def get_context(args):
     )
     parser.add_argument(
         "--dry-run",
-        default=os.environ.get("DRY_RUN", "0"),
+        default=cast_to_bool(os.environ.get("DRY_RUN", "0")),
         type=bool,
+        action=argparse.BooleanOptionalAction,
         help="Dry run mode",
     )
+    parser.add_argument(
+        "--fake",
+        default=cast_to_bool(os.environ.get("FAKE", "0")),
+        type=bool,
+        action=argparse.BooleanOptionalAction,
+        help="Marks the migrations as applied, "
+        "but without actually running the SQL to change your database schema.",
+    )
+    parser.add_argument(
+        "--migrations",
+        default=default_migrations.split(",") if default_migrations else [],
+        type=str,
+        nargs="+",
+        help="Explicit list of migrations to apply. "
+        "Specify file name, file stem or migration version like 001_init.sql, 002_test2, 003, 4",
+    )
     parser.add_argument(
         "--secure",
-        default=os.environ.get("SECURE", "0"),
+        default=cast_to_bool(os.environ.get("SECURE", "0")),
         type=bool,
-        help="Secure connection",
+        action=argparse.BooleanOptionalAction,
+        help="Use secure connection",
     )
 
     return parser.parse_args(args)
 
 
-def migrate(ctx) -> int:
-    logging.basicConfig(level=ctx.log_level, style="{", format="{levelname}:{message}")
-
-    cluster = ClickhouseCluster(
+def create_cluster(ctx) -> ClickhouseCluster:
+    return ClickhouseCluster(
         db_host=ctx.db_host,
         db_port=ctx.db_port,
         db_user=ctx.db_user,
         db_password=ctx.db_password,
         db_url=ctx.db_url,
         secure=ctx.secure,
     )
-    cluster.migrate(
+
+
+def do_migrate(cluster, ctx) -> List[Migration]:
+    return cluster.migrate(
         db_name=ctx.db_name,
         migration_path=ctx.migrations_dir,
+        explicit_migrations=ctx.migrations,
         cluster_name=ctx.cluster_name,
         multi_statement=ctx.multi_statement,
         dryrun=ctx.dry_run,
+        fake=ctx.fake,
     )
-    return 0
+
+
+def do_query_applied_migrations(cluster, ctx) -> List[Migration]:
+    with cluster.connection(ctx.db_name) as conn:
+        migrator = Migrator(conn, True)
+        return migrator.query_applied_migrations()
+
+
+def migrate(ctx) -> List[Migration]:
+    logging.basicConfig(level=ctx.log_level, style="{", format="{levelname}:{message}")
+
+    cluster = create_cluster(ctx)
+    migrations = do_migrate(cluster, ctx)
+    return migrations
 
 
 def main() -> int:
-    return migrate(get_context(sys.argv[1:]))  # pragma: no cover
+    migrate(get_context(sys.argv[1:]))  # pragma: no cover
+    return 0  # pragma: no cover

src/clickhouse_migrations/migration.py

@@ -2,14 +2,14 @@
 import os
 from collections import namedtuple
 from pathlib import Path
-from typing import List
+from typing import List, Optional, Union
 
 Migration = namedtuple("Migration", ["version", "md5", "script"])
 
 
 class MigrationStorage:
-    def __init__(self, storage_dir: Path):
-        self.storage_dir: Path = storage_dir
+    def __init__(self, storage_dir: Union[Path, str]):
+        self.storage_dir: Path = Path(storage_dir)
 
     def filenames(self) -> List[Path]:
         l: List[Path] = []
@@ -19,17 +19,28 @@ def filenames(self) -> List[Path]:
 
         return l
 
-    def migrations(self) -> List[Migration]:
+    def migrations(
+        self, explicit_migrations: Optional[List[str]] = None
+    ) -> List[Migration]:
         migrations: List[Migration] = []
 
         for full_path in self.filenames():
+            version_string = full_path.name.split("_")[0]
+            version_number = int(version_string)
             migration = Migration(
-                version=int(full_path.name.split("_")[0]),
+                version=version_number,
                 script=str(full_path.read_text(encoding="utf8")),
                 md5=hashlib.md5(full_path.read_bytes()).hexdigest(),
             )
 
-            migrations.append(migration)
+            if (
+                not explicit_migrations
+                or full_path.name in explicit_migrations
+                or full_path.stem in explicit_migrations
+                or version_string in explicit_migrations
+                or str(version_number) in explicit_migrations
+            ):
+                migrations.append(migration)
 
         migrations.sort(key=lambda m: m.version)