diff --git a/dbbackup/VERSION b/dbbackup/VERSION index 6aba2b24..fae6e3d0 100644 --- a/dbbackup/VERSION +++ b/dbbackup/VERSION @@ -1 +1 @@ -4.2.0 +4.2.1 diff --git a/dbbackup/db/sqlite.py b/dbbackup/db/sqlite.py index 1d9e3f39..a372e2b3 100644 --- a/dbbackup/db/sqlite.py +++ b/dbbackup/db/sqlite.py @@ -71,11 +71,23 @@ def restore_dump(self, dump): if not self.connection.is_usable(): self.connection.connect() cursor = self.connection.cursor() + sql_command = b"" + sql_is_complete = True for line in dump.readlines(): - try: - cursor.execute(line.decode("UTF-8")) - except (OperationalError, IntegrityError) as err: - warnings.warn(f"Error in db restore: {err}") + sql_command = sql_command + line + line_str = line.decode("UTF-8") + if line_str.startswith("INSERT") and not line_str.endswith(");\n"): + sql_is_complete = False + continue + if not sql_is_complete and line_str.endswith(");\n"): + sql_is_complete = True + + if sql_is_complete: + try: + cursor.execute(sql_command.decode("UTF-8")) + except (OperationalError, IntegrityError) as err: + warnings.warn(f"Error in db restore: {err}") + sql_command = b"" class SqliteCPConnector(BaseDBConnector): diff --git a/dbbackup/management/commands/dbrestore.py b/dbbackup/management/commands/dbrestore.py index 8e7fff97..a8ddef27 100644 --- a/dbbackup/management/commands/dbrestore.py +++ b/dbbackup/management/commands/dbrestore.py @@ -15,6 +15,7 @@ class Command(BaseDbBackupCommand): help = "Restore a database backup from storage, encrypted and/or compressed." content_type = "db" + no_drop = False option_list = BaseDbBackupCommand.option_list + ( make_option("-d", "--database", help="Database to restore"), @@ -52,6 +53,13 @@ class Command(BaseDbBackupCommand): default=[], help="Specify schema(s) to restore. Can be used multiple times.", ), + make_option( + "-r", + "--no-drop", + action="store_true", + default=False, + help="Don't clean (drop) the database. This only works with mongodb and postgresql.", + ), ) def handle(self, *args, **options): @@ -74,6 +82,7 @@ def handle(self, *args, **options): self.input_database_name ) self.storage = get_storage() + self.no_drop = options.get("no_drop") self.schemas = options.get("schema") self._restore_backup() except StorageError as err: @@ -133,8 +142,7 @@ def _restore_backup(self): input_file.seek(0) self.connector = get_connector(self.database_name) - if self.schemas: self.connector.schemas = self.schemas - self.connector.restore_dump(input_file) + self.connector.drop = not self.no_drop diff --git a/dbbackup/tests/settings.py b/dbbackup/tests/settings.py index 0e3501a7..c31a8ea3 100644 --- a/dbbackup/tests/settings.py +++ b/dbbackup/tests/settings.py @@ -29,6 +29,7 @@ "dbbackup", "dbbackup.tests.testapp", ) +DEFAULT_AUTO_FIELD = "django.db.models.AutoField" DATABASES = { "default": { diff --git a/dbbackup/tests/test_connectors/test_sqlite.py b/dbbackup/tests/test_connectors/test_sqlite.py index b448fb34..6ac737ea 100644 --- a/dbbackup/tests/test_connectors/test_sqlite.py +++ b/dbbackup/tests/test_connectors/test_sqlite.py @@ -5,7 +5,7 @@ from django.test import TestCase from dbbackup.db.sqlite import SqliteConnector, SqliteCPConnector -from dbbackup.tests.testapp.models import CharModel +from dbbackup.tests.testapp.models import CharModel, TextModel class SqliteConnectorTest(TestCase): @@ -28,7 +28,17 @@ def test_create_dump_with_unicode(self): dump = connector.create_dump() self.assertTrue(dump.read()) + def test_create_dump_with_newline(self): + TextModel.objects.create( + field=f'INSERT ({"foo" * 5000}\nbar\n WHERE \nbaz IS\n "great" );\n' + ) + + connector = SqliteConnector() + dump = connector.create_dump() + self.assertTrue(dump.read()) + def test_restore_dump(self): + TextModel.objects.create(field="T\nf\nw\nnl") connector = SqliteConnector() dump = connector.create_dump() connector.restore_dump(dump) diff --git a/dbbackup/tests/testapp/migrations/0001_initial.py b/dbbackup/tests/testapp/migrations/0001_initial.py index b3fb64c0..1febcd9c 100644 --- a/dbbackup/tests/testapp/migrations/0001_initial.py +++ b/dbbackup/tests/testapp/migrations/0001_initial.py @@ -3,6 +3,7 @@ class Migration(migrations.Migration): + initial = True dependencies = [] operations = [ diff --git a/dbbackup/tests/testapp/migrations/0002_textmodel.py b/dbbackup/tests/testapp/migrations/0002_textmodel.py new file mode 100644 index 00000000..ecc159c2 --- /dev/null +++ b/dbbackup/tests/testapp/migrations/0002_textmodel.py @@ -0,0 +1,28 @@ +# Generated by Django 4.0.1 on 2022-04-27 22:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("testapp", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="TextModel", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("field", models.TextField()), + ], + ), + ] diff --git a/dbbackup/tests/testapp/models.py b/dbbackup/tests/testapp/models.py index fcb5efa1..c68f101f 100644 --- a/dbbackup/tests/testapp/models.py +++ b/dbbackup/tests/testapp/models.py @@ -1,22 +1,14 @@ from django.db import models -___all__ = ( - "CharModel", - "IntegerModel", - "TextModel", - "BooleanModel" "DateModel", - "DateTimeModel", - "ForeignKeyModel", - "ManyToManyModel", - "FileModel", - "TestModel", -) - class CharModel(models.Model): field = models.CharField(max_length=10) +class TextModel(models.Model): + field = models.TextField() + + class ForeignKeyModel(models.Model): field = models.ForeignKey(CharModel, on_delete=models.CASCADE) diff --git a/docs/changelog.rst b/docs/changelog.rst index 88fbda35..27150090 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,12 @@ Unreleased * Nothing (yet)! +4.2.1 (2024-08-23) +---------- + +* Add --no-drop option to dbrestore command to prevent dropping tables before restoring data. +* Fix bug where sqlite dbrestore would fail if field data contains the line break character. + 4.2.0 (2024-08-22) ------------------ @@ -20,91 +26,57 @@ Unreleased 4.1.0 (2024-01-14) ------------------ -* Fix restore fail after editing filename by @stan-levend in https://github.com/jazzband/django-dbbackup/pull/465 -* Drop python 3.6 by @Archmonger in https://github.com/jazzband/django-dbbackup/pull/472 -* update links by @Arhell in https://github.com/jazzband/django-dbbackup/pull/471 -* Update doc for backup directory consistency by @jej in https://github.com/jazzband/django-dbbackup/pull/473 -* RESTORE_PREFIX for RESTORE_SUFFIX by @mativs in https://github.com/jazzband/django-dbbackup/pull/469 -* Support Django 4.1, 4.2 and Python 3.11 by @johnthagen in https://github.com/jazzband/django-dbbackup/pull/485 -* Support Python 3.12 and Django 5.0 by @johnthagen in https://github.com/jazzband/django-dbbackup/pull/499 +* Fix restore fail after editing filename +* Drop python 3.6 +* update links +* Update doc for backup directory consistency +* RESTORE_PREFIX for RESTORE_SUFFIX +* Support Django 4.1, 4.2 and Python 3.11 +* Support Python 3.12 and Django 5.0 4.0.2 (2022-09-27) ------------------ -* support for prometheus wrapped dbs by @tsundokum in https://github.com/jazzband/django-dbbackup/pull/455 -* Backup of SQLite fail if there are Virtual Tables (e.g. FTS tables). by @xbello in https://github.com/jazzband/django-dbbackup/pull/458 -* Closes #460: python-gnupg version increase breaks unencrypt_file func… by @chambersh1129 in https://github.com/jazzband/django-dbbackup/pull/461 +* support for prometheus wrapped dbs +* Backup of SQLite fail if there are Virtual Tables (e.g. FTS tables). +* Closes #460: python-gnupg version increase breaks unencrypt_file func… 4.0.1 (2022-07-09) --------------------- * As of this version, dbbackup is now within Jazzband! This version tests our Jazzband release CI, and adds miscellaneous refactoring/cleanup. -* Fix GitHub Actions configuration by @johnthagen in https://github.com/jazzband/django-dbbackup/pull/419 -* Enable functional tests in CI by @johnthagen in https://github.com/jazzband/django-dbbackup/pull/420 -* Update settings.py comment by @aaronvarghese in https://github.com/jazzband/django-dbbackup/pull/427 -* Jazzband transfer tasks by @Archmonger in https://github.com/jazzband/django-dbbackup/pull/418 -* Refactoring and tooling by @Archmonger in https://github.com/jazzband/django-dbbackup/pull/438 +* Fix GitHub Actions configuration +* Enable functional tests in CI +* Update settings.py comment +* Jazzband transfer tasks +* Refactoring and tooling 4.0.0b0 (2021-12-19) -------------------- -* Fix RemovedInDjango41Warning related to default_app_config `#413`_ -* Add authentication database support for MongoDB `#379`_ -* Remove six dependency `#371`_ -* Explicitly support Python 3.6+. `#408`_ -* Drop support for end of life Django versions. Currently support 2.2, 3.2, 4.0. `#408`_ -* Replace ugettext_lazy with gettext_lazy `#342`_ -* Changed logging settings from settings.py to late init `#332`_ -* Fix authentication error when postgres is password protected `#361`_ -* Use exclude-table-data instead of exclude-table `#363`_ -* Add support for exclude tables data in the command interface `#375`_ -* Move author and version information into setup.py to allow building package in isolated - environment (e.g. with the ``build`` package). `#414`_ -* Documentation fixes `#341`_ `#333`_ `#349`_ `#348`_ `#337`_ `#411`_ +* Fix RemovedInDjango41Warning related to default_app_config +* Add authentication database support for MongoDB +* Remove six dependency +* Explicitly support Python 3.6+. +* Drop support for end of life Django versions. Currently support 2.2, 3.2, 4.0. +* Replace ugettext_lazy with gettext_lazy +* Changed logging settings from settings.py to late init +* Fix authentication error when postgres is password protected +* Use exclude-table-data instead of exclude-table +* Add support for exclude tables data in the command interface +* Move author and version information into setup.py to allow building package in isolated environment (e.g. with the ``build`` package). +* Documentation fixes 3.3.0 (2020-04-14) ------------------ -* Documentation fixes `#341`_ `#333`_ `#328`_ `#320`_ `#305`_ `#303`_ `#302`_ `#298`_ `#281`_ `#266`_ `#349`_ `#348`_ `#337`_ -* "output-filename" in mediabackup command `#324`_ -* Fixes for test infrastructure and mongodb support `#318`_ -* sqlite3: don't throw warnings if table already exists `#317`_ -* Fixes for django3 and updated travis (and File handling) `#316`_ -* Restoring from FTP `#313`_ -* Fixes to run dbbackup management command in Postgres for non-latin Windows. `#273`_ -* Apply changes from pull request 244; Update to include sftp storage `#280`_ -* Quick fix for proper selection of DB name to restore `#260`_ - -.. _`#342`: https://github.com/jazzband/django-dbbackup/pull/342 -.. _`#332`: https://github.com/jazzband/django-dbbackup/pull/332 -.. _`#361`: https://github.com/jazzband/django-dbbackup/pull/361 -.. _`#363`: https://github.com/jazzband/django-dbbackup/pull/363 -.. _`#375`: https://github.com/jazzband/django-dbbackup/pull/375 -.. _`#341`: https://github.com/jazzband/django-dbbackup/pull/341 -.. _`#333`: https://github.com/jazzband/django-dbbackup/pull/333 -.. _`#328`: https://github.com/jazzband/django-dbbackup/pull/328 -.. _`#320`: https://github.com/jazzband/django-dbbackup/pull/320 -.. _`#305`: https://github.com/jazzband/django-dbbackup/pull/305 -.. _`#303`: https://github.com/jazzband/django-dbbackup/pull/303 -.. _`#302`: https://github.com/jazzband/django-dbbackup/pull/302 -.. _`#298`: https://github.com/jazzband/django-dbbackup/pull/298 -.. _`#281`: https://github.com/jazzband/django-dbbackup/pull/281 -.. _`#266`: https://github.com/jazzband/django-dbbackup/pull/266 -.. _`#324`: https://github.com/jazzband/django-dbbackup/pull/324 -.. _`#318`: https://github.com/jazzband/django-dbbackup/pull/318 -.. _`#317`: https://github.com/jazzband/django-dbbackup/pull/317 -.. _`#316`: https://github.com/jazzband/django-dbbackup/pull/316 -.. _`#313`: https://github.com/jazzband/django-dbbackup/pull/313 -.. _`#273`: https://github.com/jazzband/django-dbbackup/pull/273 -.. _`#280`: https://github.com/jazzband/django-dbbackup/pull/280 -.. _`#260`: https://github.com/jazzband/django-dbbackup/pull/260 -.. _`#349`: https://github.com/jazzband/django-dbbackup/pull/349 -.. _`#348`: https://github.com/jazzband/django-dbbackup/pull/348 -.. _`#337`: https://github.com/jazzband/django-dbbackup/pull/337 -.. _`#408`: https://github.com/jazzband/django-dbbackup/pull/408 -.. _`#371`: https://github.com/jazzband/django-dbbackup/pull/371 -.. _`#379`: https://github.com/jazzband/django-dbbackup/pull/379 -.. _`#411`: https://github.com/jazzband/django-dbbackup/pull/411 -.. _`#413`: https://github.com/jazzband/django-dbbackup/pull/413 -.. _`#414`: https://github.com/jazzband/django-dbbackup/pull/414 +* Documentation fixes +* "output-filename" in mediabackup command +* Fixes for test infrastructure and mongodb support +* sqlite3: don't throw warnings if table already exists +* Fixes for django3 and updated travis (and File handling) +* Restoring from FTP +* Fixes to run dbbackup management command in Postgres for non-latin Windows. +* Apply changes from pull request 244; Update to include sftp storage +* Quick fix for proper selection of DB name to restore diff --git a/docs/conf.py b/docs/conf.py index 02c855e2..a85fe313 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -45,7 +45,7 @@ # General information about the project. project = "django-dbbackup" -copyright = "2016, Michael Shepanski" +copyright = "Mark Bakhit" # basepath path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -102,11 +102,8 @@ # -- Options for HTML output --------------------------------------------------- -on_rtd = os.environ.get("READTHEDOCS", None) == "True" -if on_rtd: - html_theme = "default" -else: - html_theme = "nature" +html_theme = "sphinx_rtd_theme" + # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/index.rst b/docs/index.rst index 236dcdb0..a392bace 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,13 +15,6 @@ It is made to: - Keep your development database up to date - Use Crontab or Celery to setup automated backups -.. warning:: - - Django DBBackup version 3 is very different to its predecessors. - See `Upgrade documentation`_ to help to get up to date. - -.. _`Upgrade documentation`: upgrade.html - Contents: .. toctree:: @@ -33,7 +26,6 @@ Contents: storage commands integration - upgrade contributing changelog diff --git a/docs/installation.rst b/docs/installation.rst index e9d4e831..e26c5015 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -24,7 +24,7 @@ that are at least present when using PyPi repositories. :: - pip install -e git+https://github.com/mjs7231/django-dbbackup.git#egg=django-dbbackup + pip install -e git+https://github.com/jazzband/django-dbbackup.git#egg=django-dbbackup Add it in your project diff --git a/docs/upgrade.rst b/docs/upgrade.rst deleted file mode 100644 index 1a84485c..00000000 --- a/docs/upgrade.rst +++ /dev/null @@ -1,98 +0,0 @@ -Upgrade from 2.5.x -================== - -Settings --------- - -The following settings are now useless, you can remove them: - -- ``DBBACKUP_BACKUP_ENVIRONMENT``: Must be set in ``CONNECTORS['dbname']['env']`` -- ``DBBACKUP_RESTORE_ENVIRONMENT``: Same as ``BACKUP_ENVIRONMENT`` -- ``DBBACKUP_FORCE_ENGINE`` -- ``DBBACKUP_READ_FILE`` -- ``DBBACKUP_WRITE_FILE`` -- ``DBBACKUP_BACKUP_DIRECTORY``: Was used by Filesystem storage, use - ``location`` parameter -- ``DBBACKUP_SQLITE_BACKUP_COMMANDS``: Was used by SQLite database, use - ``CONNECTORS``'s parameters. -- ``DBBACKUP_SQLITE_RESTORE_COMMANDS``: Same as ``SQLITE_BACKUP_COMMANDS`` -- ``DBBACKUP_MYSQL_BACKUP_COMMANDS``: Same as ``SQLITE_BACKUP_COMMANDS`` but - for MySQL -- ``DBBACKUP_MYSQL_RESTORE_COMMANDS``: Same as ``MYSQL_BACKUP_COMMANDS`` -- ``DBBACKUP_POSTGRESQL_BACKUP_COMMANDS`` Same as ``MYSQL_BACKUP_COMMANDS`` - but for PostgreSQL -- ``DBBACKUP_POSTGRESQL_RESTORE_COMMANDS``: Same as - ``DBBACKUP_POSTGRESQL_BACKUP_COMMANDS``: Was used to activate PostGIS, use - ``PgDumpGisConnector`` connector to enable this option -- ``DBBACKUP_POSTGRESQL_RESTORE_SINGLE_TRANSACTION``: Must be set in - ``CONNTECTORS['dbname']['single_transaction']`` -- ``DBBACKUP_BUILTIN_STORAGE`` - -Commands --------- - -dbrestore -~~~~~~~~~ - -``--backup-extension`` has been removed, DBBackup should automatically -know the appropriate file. - -Listing from this command, ``--list``, has been removed in favor of -``listbackups`` command. - -mediabackup -~~~~~~~~~~~ - -``mediabackup``'s ``--no-compress`` option has been replaced by ``--compress`` -to maintain consistency with other commands. - -This command can now backup remote storage, not only the local filesystem -``DBBACKUP_BACKUP_DIRECTORY``. - -mediarestore -~~~~~~~~~~~~ - -You are now able to restore your media files backups. Unfortunately you'll not -be able to restore backup files from previous versions of dbbackup. - -Database connector ------------------- - -We made a total refactoring of DBCommands system. It is now easier to use, -configure and implement a custom connector. - -All database configuration for backups are defined in settings -``DBBACKUP_CONNECTORS``. By default, the ``DATABASES`` -parameters are used but can be overridden in this new constant. - -This dictionary stores configuration about how backups are made, - the path of the backup command (``/bin/mysqldump``), add a suffix or a prefix -to the command line, environment variable, etc. - -The system has been kept pretty simple and can detect alone how to backup your DB. -If it can't just submit to us your Django DB Engine and we'll try to fix -it. - -SQLite -~~~~~~ - -Previously a backup was made by copying the database file. Now you have the choice -between making a raw snapshot or making a real SQL dump. It can be useful to exclude -tables or to not overwrite data. - -If you want to restore your old backups choose -``dbbackup.db.sqlite.SqliteCPConnector``. - - -Storage engine --------------- - -All storage engines has been removed from DBBackup except the basic filesystem storage. -The project now uses Django storages as an intermediary driver. - -``settings.DBBACKUP_STORAGE`` must now be a full path to a Django storage, for -example ``'django.core.files.storage.FileSystemStorage'``. -``settings.DBBACKUP_STORAGE_OPTIONS`` maintains its function of containing the options of the storage. - -If you were using a storage backend that has been removed, don't worry, we will ensure you -have a solution by testing and writing an equivalent using Django-Storages. diff --git a/requirements/docs.txt b/requirements/docs.txt index d1fd12c9..96690afd 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,4 +1,6 @@ +. docutils python-dotenv sphinx sphinx-django-command +sphinx-rtd-theme