From 2c2e5b6e7752ebeb662373c3469d7930f4840fd0 Mon Sep 17 00:00:00 2001 From: Mischa ter Smitten Date: Fri, 1 Jul 2022 17:00:14 +0200 Subject: [PATCH] Handle backups made with newer versions of mydumper (#10) * Handle backups made with newer versions of mydumper * Cleanup * Fix test * Hardcoded fix * Improved tests * Improved tests * Fix failing tests * More stict * Cs fixes (#11) * Cs fixes * Cs fixes * Fix requirements * Cs fixes * Cleanup --- .travis.yml | 12 +++--- Makefile | 12 ++++-- README.md | 1 + bin/randomize-ids | 2 + pylintrc | 16 ++++++- requirements-dev.txt | 5 ++- setup.py | 13 +++++- untraceables/test/test_configuration.py | 15 +++++-- untraceables/test/test_file.py | 12 +++++- untraceables/test/test_filter.py | 37 ++++++++++------ untraceables/test/test_formatter.py | 33 +++++++++------ untraceables/test/test_mysql.py | 56 +++++++++++++++++++------ untraceables/test/test_query.py | 8 ++++ untraceables/test/test_validation.py | 8 ++++ untraceables/utilities/formatter.py | 3 +- untraceables/utilities/query.py | 10 +++-- 16 files changed, 184 insertions(+), 59 deletions(-) diff --git a/.travis.yml b/.travis.yml index 93155d0..f5a6752 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,16 +3,18 @@ language: python python: - "2.7" - "3.5" + - "3.6" services: - mysql before_install: - - mysql -uroot -e 'CREATE DATABASE IF NOT EXISTS `untraceables_test`;' - - mysql -uroot -e "CREATE USER 'untraceables'@'localhost' IDENTIFIED BY 'mmRXHqnc3zSshYjxSv8n';" - - mysql -uroot -e "GRANT ALL PRIVILEGES ON untraceables_test . * TO 'untraceables'@'localhost';" - - mysql -uroot -D untraceables_test -e 'CREATE TABLE `users` (`id` int(10) unsigned NOT NULL, `mapped_id` int(10) unsigned NOT NULL, PRIMARY KEY (`id`), UNIQUE KEY `mapped_id` (`mapped_id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8;'; - - mysql -uroot -D untraceables_test -e 'INSERT INTO `users` (`id`, `mapped_id`) VALUES (0, 10), (1, 9), (2, 8);'; + - | + mysql -uroot -e 'CREATE DATABASE IF NOT EXISTS `untraceables_test`;' + mysql -uroot -e "CREATE USER 'untraceables'@'localhost' IDENTIFIED BY 'mmRXHqnc3zSshYjxSv8n';" + mysql -uroot -e "GRANT ALL PRIVILEGES ON untraceables_test . * TO 'untraceables'@'localhost';" + mysql -uroot -D untraceables_test -e 'CREATE TABLE `users` (`id` int(10) unsigned NOT NULL, `mapped_id` int(10) unsigned NOT NULL, PRIMARY KEY (`id`), UNIQUE KEY `mapped_id` (`mapped_id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8;'; + mysql -uroot -D untraceables_test -e 'INSERT INTO `users` (`id`, `mapped_id`) VALUES (0, 10), (1, 9), (2, 8);'; install: make init-dev diff --git a/Makefile b/Makefile index 01d072a..001f888 100644 --- a/Makefile +++ b/Makefile @@ -9,9 +9,15 @@ init-dev: source: python setup.py sdist -check: - find . -name \*.py | xargs pycodestyle --first - find bin -type f | xargs pycodestyle --first +pycodestyle: + find . -type f -name \*.py | xargs --no-run-if-empty pycodestyle --first + find bin -type f | xargs --no-run-if-empty pycodestyle --first + +pylint: + find . -type f -name \*.py | xargs --no-run-if-empty pylint + find bin -type f | xargs --no-run-if-empty pylint + +check: pycodestyle pylint test: nosetests -v diff --git a/README.md b/README.md index 7a4a56b..46c4701 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ * Python 2.7 * Python 3.5 +* Python 3.6 ## Usage diff --git a/bin/randomize-ids b/bin/randomize-ids index a85f4fd..53c7898 100755 --- a/bin/randomize-ids +++ b/bin/randomize-ids @@ -6,6 +6,8 @@ Randomizes IDs for a given set of tables making them untraceable across environments """ +# pylint: disable=invalid-name + from __future__ import print_function from __future__ import absolute_import import argparse diff --git a/pylintrc b/pylintrc index 9c93cce..25c9a79 100644 --- a/pylintrc +++ b/pylintrc @@ -1,8 +1,20 @@ +[MASTER] +extension-pkg-whitelist=MySQLdb + [FORMAT] max-line-length=120 indent-string=' ' -[TYPECHECK] -ignored-classes=MySQLdb +[SIMILARITIES] +min-similarity-lines=4 +ignore-comments=yes +ignore-docstrings=yes +ignore-imports=yes + +[MESSAGES CONTROL] +disable=broad-except,too-many-locals + +[BASIC] +good-names=f,e diff --git a/requirements-dev.txt b/requirements-dev.txt index 04aa2e9..3664c6c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,5 +2,6 @@ configobj==5.0.6 mycli==1.20.1 mysqlclient==1.4.6 nose==1.3.7 -pycodestyle==2.5.0 -pylint==1.9.5 +pycodestyle==2.6.0 +pylint==2.6.2 ; python_version >= '3' +pylint==1.9.5 ; python_version < '3' diff --git a/setup.py b/setup.py index 071e48f..39667ae 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,21 @@ # -*- coding: utf-8 -*- +""" +The setup script is the centre of all activity in building, distributing, and installing modules. +""" + from __future__ import absolute_import from setuptools import setup, find_packages def readme(): - with open('README.md') as f: - return f.read() + """ + Return README content. + + :return: The specified number of bytes from the file + """ + with open('README.md') as filepointer: + return filepointer.read() setup(name='untraceables', diff --git a/untraceables/test/test_configuration.py b/untraceables/test/test_configuration.py index 5df3bba..b5851e1 100644 --- a/untraceables/test/test_configuration.py +++ b/untraceables/test/test_configuration.py @@ -1,5 +1,9 @@ # -*- coding: utf-8 -*- +""" +Tests for `configuration_utility`. +""" + from __future__ import absolute_import import os import unittest @@ -8,6 +12,9 @@ class TestConfiguration(unittest.TestCase): + """ + TestCase. + """ def test_read_file(self): """ @@ -29,7 +36,7 @@ def test_read_file(self): expected = 'dolor' self.assertEqual(expected, actual['main']['password']) - def test_read_file(self): + def test_read_file_0(self): """ Tests `read_xclude_regexes_file`. @@ -40,7 +47,7 @@ def test_read_file(self): actual = configuration_utility.read_xclude_regexes_file('lorem') self.assertEqual(expected, actual) - def test_read_file_0(self): + def test_read_file_1(self): """ Tests `read_xclude_regexes_file`. @@ -56,8 +63,9 @@ def test_read_file_0(self): r'^users\.user_id$', r'^users\..*user_id$'] actual = configuration_utility.read_xclude_regexes_file(filename) + self.assertEqual(expected, actual) - def test_read_file_1(self): + def test_read_file_2(self): """ Tests `read_xclude_regexes_file`. @@ -67,6 +75,7 @@ def test_read_file_1(self): filename = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'include-from-1') expected = [] actual = configuration_utility.read_xclude_regexes_file(filename) + self.assertEqual(expected, actual) suite = unittest.TestLoader().loadTestsFromTestCase(TestConfiguration) diff --git a/untraceables/test/test_file.py b/untraceables/test/test_file.py index 06304e5..17aba2c 100644 --- a/untraceables/test/test_file.py +++ b/untraceables/test/test_file.py @@ -1,12 +1,20 @@ # -*- coding: utf-8 -*- +""" +Tests for `file_utility`. +""" + from __future__ import absolute_import import os import unittest + from untraceables.utilities import file as file_utility class TestFile(unittest.TestCase): + """ + TestCase. + """ def test_get_sorted_file_list(self): """ @@ -18,12 +26,12 @@ def test_get_sorted_file_list(self): 'test_split_file_0.sql', 'test_split_file_1.sql', 'test_split_file_2.sql', 'untraceables.cfg'] actual = file_utility.get_sorted_file_list(path) - self.assertEquals(expected, list(actual)) + self.assertEqual(expected, list(actual)) path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data-non-existing') expected = [] actual = file_utility.get_sorted_file_list(path) - self.assertEquals(expected, list(actual)) + self.assertEqual(expected, list(actual)) suite = unittest.TestLoader().loadTestsFromTestCase(TestFile) diff --git a/untraceables/test/test_filter.py b/untraceables/test/test_filter.py index 5a47687..1536e07 100644 --- a/untraceables/test/test_filter.py +++ b/untraceables/test/test_filter.py @@ -1,11 +1,19 @@ # -*- coding: utf-8 -*- +""" +Tests for `filter_utility`. +""" + from __future__ import absolute_import import unittest + from untraceables.utilities import filter as filter_utility class TestFilter(unittest.TestCase): + """ + TestCase. + """ def test_show_tables(self): """ @@ -16,34 +24,34 @@ def test_show_tables(self): inclusive_regexes = exclusive_regexes = [] expected = set([]) actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes) - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) table_columns = iter(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing']) inclusive_regexes = exclusive_regexes = [] expected = set([]) actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes) - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) table_columns = iter(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing']) inclusive_regexes = [r'^.*\..*$'] exclusive_regexes = [] expected = set(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing']) actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes) - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) table_columns = iter(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing']) inclusive_regexes = [r'^.*\..*$'] exclusive_regexes = [r'^sit\..*$'] expected = set(['ipsum.dolor', 'consectetur.adipiscing']) actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes) - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) table_columns = iter(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing']) inclusive_regexes = [r'^.*\..*$'] exclusive_regexes = [r'^.*\.adipiscing$'] expected = set(['ipsum.dolor', 'sit.amet']) actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes) - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) def test_table_names_from_mydumper_backup(self): """ @@ -55,22 +63,27 @@ def test_table_names_from_mydumper_backup(self): expected = [] actual = filter_utility.table_names_from_mydumper_backup(files, suffixed_database) self.assertTrue(hasattr(actual, 'next') or hasattr(actual, '__next__')) - self.assertEquals(expected, list(actual)) + self.assertEqual(expected, list(actual)) files = ['ipsum.dolor-schema.sql', 'ipsum.dolor.sql', - 'ipsum.consectetur-schema.sql', 'ipsum.consectetur.sql'] + 'ipsum.consectetur-schema.sql', 'ipsum.consectetur.sql', + 'ipsum-schema-create.sql', 'ipsum.sit-schema.sql', 'ipsum.sit.00000.sql', + 'ipsum.sit.00001.sql', 'ipsum.sit.12356.sql', 'ipsum.sit.99999.sql'] suffixed_database = 'ipsum.' - expected = ['ipsum.dolor.sql', 'ipsum.consectetur.sql'] + expected = ['ipsum.dolor.sql', 'ipsum.consectetur.sql', 'ipsum.sit.00000.sql', + 'ipsum.sit.00001.sql', 'ipsum.sit.12356.sql', 'ipsum.sit.99999.sql'] actual = filter_utility.table_names_from_mydumper_backup(files, suffixed_database) self.assertTrue(hasattr(actual, 'next') or hasattr(actual, '__next__')) - self.assertEquals(expected, list(actual)) + self.assertEqual(expected, list(actual)) files = ['ipsum.dolor-schema.sql', 'ipsum.dolor.sql', - 'consectetur.adipiscing-schema.sql', 'consectetur.adipiscing.sql'] + 'ipsum-schema-create.sql', 'ipsum.sit-schema.sql', 'ipsum.sit.00000.sql', + 'consectetur.adipiscing-schema.sql', 'consectetur.adipiscing.sql', + 'elit-schema-create.sql', 'elit.mollis-schema.sql', 'elit.mollis.00000.sql'] suffixed_database = 'ipsum.' - expected = ['ipsum.dolor.sql'] + expected = ['ipsum.dolor.sql', 'ipsum.sit.00000.sql'] actual = filter_utility.table_names_from_mydumper_backup(files, suffixed_database) - self.assertEquals(expected, list(actual)) + self.assertEqual(expected, list(actual)) suite = unittest.TestLoader().loadTestsFromTestCase(TestFilter) diff --git a/untraceables/test/test_formatter.py b/untraceables/test/test_formatter.py index 181ec80..fe12d3e 100644 --- a/untraceables/test/test_formatter.py +++ b/untraceables/test/test_formatter.py @@ -1,11 +1,19 @@ # -*- coding: utf-8 -*- +""" +Tests for `formatter_utility`. +""" + from __future__ import absolute_import import unittest + from untraceables.utilities import formatter as formatter_utility class TestFormatter(unittest.TestCase): + """ + TestCase. + """ def test_show_tables(self): """ @@ -16,17 +24,17 @@ def test_show_tables(self): expected = iter([]) actual = formatter_utility.show_tables(table_columns) self.assertTrue(hasattr(actual, 'next') or hasattr(actual, '__next__')) - self.assertEquals(list(expected), list(actual)) + self.assertEqual(list(expected), list(actual)) table_columns = ({'TABLE_NAME': 'lorem', 'COLUMN_NAME': 'ipsum'},) expected = iter(['lorem.ipsum']) actual = formatter_utility.show_tables(table_columns) - self.assertEquals(list(expected), list(actual)) + self.assertEqual(list(expected), list(actual)) table_columns = ({'TABLE_NAME': 'lorem', 'COLUMN_NAME': 'ipsum'}, {'TABLE_NAME': 'dolor', 'COLUMN_NAME': 'sit'}) expected = iter(['lorem.ipsum', 'dolor.sit']) actual = formatter_utility.show_tables(table_columns) - self.assertEquals(list(expected), list(actual)) + self.assertEqual(list(expected), list(actual)) def test_table_columns_tsv(self): """ @@ -37,7 +45,7 @@ def test_table_columns_tsv(self): table_columns = ['lorem.ipsum', 'dolor.sit'] expected = ['adipiscing\tlorem\tipsum', 'adipiscing\tdolor\tsit'] actual = formatter_utility.table_columns_tsv(database, table_columns) - self.assertEquals(list(expected), list(actual)) + self.assertEqual(list(expected), list(actual)) def test_randomize_queries(self): """ @@ -48,12 +56,12 @@ def test_randomize_queries(self): expected = ('SELECT NOW();\n' 'SELECT 1;\n') actual = formatter_utility.randomize_queries(queries) - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) queries = [] expected = '' actual = formatter_utility.randomize_queries(queries) - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) def test_table_names_from_mydumper_backup(self): """ @@ -65,14 +73,15 @@ def test_table_names_from_mydumper_backup(self): actual = formatter_utility.table_names_from_mydumper_backup(files, suffixed_database) expected = [] self.assertTrue(hasattr(actual, 'next') or hasattr(actual, '__next__')) - self.assertEquals(expected, list(actual)) + self.assertEqual(expected, list(actual)) - files = ['ipsum.dolor.sql', 'ipsum.consectetur.sql'] + files = ['ipsum.dolor.sql', 'ipsum.consectetur.sql', 'ipsum.sit.00000.sql', + 'ipsum.elit.00001.sql', 'ipsum.donec.12356.sql', 'ipsum.lacus.99999.sql'] suffixed_database = 'ipsum.' actual = formatter_utility.table_names_from_mydumper_backup(files, suffixed_database) - expected = ['dolor', 'consectetur'] + expected = ['dolor', 'consectetur', 'sit', 'elit', 'donec', 'lacus'] self.assertTrue(hasattr(actual, 'next') or hasattr(actual, '__next__')) - self.assertEquals(expected, list(actual)) + self.assertEqual(expected, list(actual)) def test_inclusive_regex_in(self): """ @@ -83,7 +92,7 @@ def test_inclusive_regex_in(self): database_table_delimiter = r'\.' actual = formatter_utility.inclusive_regex_in(inclusive_regex, database_table_delimiter) expected = r'^ipsum', r'id$' - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) def test_inclusive_regex_out(self): """ @@ -95,7 +104,7 @@ def test_inclusive_regex_out(self): database_table_delimiter = r'\.' actual = formatter_utility.inclusive_regex_out(file_basename, field_regex, database_table_delimiter) expected = r'^ipsum\.id$' - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) suite = unittest.TestLoader().loadTestsFromTestCase(TestFormatter) diff --git a/untraceables/test/test_mysql.py b/untraceables/test/test_mysql.py index 595d771..6b47efe 100644 --- a/untraceables/test/test_mysql.py +++ b/untraceables/test/test_mysql.py @@ -1,10 +1,15 @@ # -*- coding: utf-8 -*- +""" +Tests for `mysql_utility`. +""" + from __future__ import absolute_import -import MySQLdb -import MySQLdb.cursors import os import unittest +import MySQLdb +import MySQLdb.cursors + from untraceables.utilities import mysql as mysql_utility MYSQL_HOST = 'localhost' @@ -29,6 +34,9 @@ class TestMySql(unittest.TestCase): + """ + TestCase. + """ def test_close_connection_and_cursor_close_not_called(self): """ @@ -48,10 +56,13 @@ def test_close_connection_and_cursor_close_called_on_connection(self): Connection has a close method. """ - connection = connection_mock() + connection = ConnectionMock() cursor = None - actual = mysql_utility.close_connection_and_cursor(connection, cursor) - self.assertRaisesRegexp(Warning, 'close called on connection') + mysql_utility.close_connection_and_cursor(connection, cursor) + try: + self.assertRaisesRegex(Warning, 'close called on connection') + except AttributeError: + self.assertRaisesRegexp(Warning, 'close called on connection') # noqa pylint: disable=deprecated-method def test_close_connection_and_cursor_close_called_on_cursor(self): """ @@ -61,9 +72,12 @@ def test_close_connection_and_cursor_close_called_on_cursor(self): """ connection = None - cursor = cursor_mock() - actual = mysql_utility.close_connection_and_cursor(connection, cursor) - self.assertRaisesRegexp(Warning, 'close called on cursor') + cursor = CursorMock() + mysql_utility.close_connection_and_cursor(connection, cursor) + try: + self.assertRaisesRegex(Warning, 'close called on cursor') + except AttributeError: + self.assertRaisesRegexp(Warning, 'close called on cursor') # noqa pylint: disable=deprecated-method def test_split_file_0(self): """ @@ -159,7 +173,7 @@ def test_get_connection_failure(self): """ try: - actual = mysql_utility.get_connection(MYSQL_HOST, MYSQL_USER, 'lorem', MYSQL_DATABASE) + mysql_utility.get_connection(MYSQL_HOST, MYSQL_USER, 'lorem', MYSQL_DATABASE) except Exception as e: self.assertIsInstance(e, MySQLdb.OperationalError) @@ -188,7 +202,7 @@ def test_get_cursor_failure(self): """ try: - actual = mysql_utility.get_cursor(None) + mysql_utility.get_cursor(None) except Exception as e: self.assertIsInstance(e, AttributeError) @@ -244,15 +258,33 @@ def test_get_max_id(self): mysql_utility.close_connection_and_cursor(connection, cursor) -class connection_mock(object): +# pylint: disable=useless-object-inheritance,no-self-use,too-few-public-methods + +class ConnectionMock(object): + """ + Mock for connection. + """ + def close(self): + """ + To check that close is called. + """ raise Warning('close called on connection') -class cursor_mock(object): +class CursorMock(object): + """ + Mock for cursor. + """ + def close(self): + """ + To check that close is called. + """ raise Warning('close called on cursor') +# pylint: enable=useless-object-inheritance,no-self-use,too-few-public-methods + suite = unittest.TestLoader().loadTestsFromTestCase(TestMySql) unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/untraceables/test/test_query.py b/untraceables/test/test_query.py index f3eb400..9c6fd4a 100644 --- a/untraceables/test/test_query.py +++ b/untraceables/test/test_query.py @@ -1,11 +1,19 @@ # -*- coding: utf-8 -*- +""" +Tests for `query_utility`. +""" + from __future__ import absolute_import import unittest + from untraceables.utilities import query as query_utility class TestQuery(unittest.TestCase): + """ + TestCase. + """ def test_get_show_table_columns(self): """ diff --git a/untraceables/test/test_validation.py b/untraceables/test/test_validation.py index e0341f6..1945eb7 100644 --- a/untraceables/test/test_validation.py +++ b/untraceables/test/test_validation.py @@ -1,11 +1,19 @@ # -*- coding: utf-8 -*- +""" +Tests for `validation_utility`. +""" + from __future__ import absolute_import import unittest + from untraceables.utilities import validation as validation_utility class TestValidation(unittest.TestCase): + """ + TestCase. + """ def test_check_max_ids(self): """ diff --git a/untraceables/utilities/formatter.py b/untraceables/utilities/formatter.py index 4b5ccd8..c88466f 100644 --- a/untraceables/utilities/formatter.py +++ b/untraceables/utilities/formatter.py @@ -6,6 +6,7 @@ from __future__ import absolute_import import os +import re def show_tables(table_columns): @@ -74,7 +75,7 @@ def table_names_from_mydumper_backup(files, suffixed_database): """ for file_name in files: - yield os.path.splitext(file_name)[0].replace(suffixed_database, '') + yield re.sub(r'\.\d+$', '', os.path.splitext(file_name)[0].replace(suffixed_database, '')) def inclusive_regex_in(inclusive_regex, database_table_delimiter): diff --git a/untraceables/utilities/query.py b/untraceables/utilities/query.py index cc3ff6c..e6db24c 100644 --- a/untraceables/utilities/query.py +++ b/untraceables/utilities/query.py @@ -85,6 +85,8 @@ def get_unique_checks(enabled): return 'SET UNIQUE_CHECKS={0:d}'.format(enabled) +# pylint: disable=too-many-arguments + def get_randomize(database, table, columns, column, mapping_database, mapping_table): """ @@ -143,11 +145,11 @@ def _get_randomize(database, table, columns, column, mapping_database, mapping_t query.append('SELECT') select = [] - for c in columns: - if c['Field'] == column: + for show_column in columns: + if show_column['Field'] == column: select.append('`t2`.`{:s}`'.format(untraceables.MAPPING_ID_FIELD)) else: - select.append('`t1`.`{:s}`'.format(c['Field'])) + select.append('`t1`.`{:s}`'.format(show_column['Field'])) query.append(', '.join(select)) query.append('FROM `{:s}`.`{:s}` `t1`'.format(database, table)) @@ -156,3 +158,5 @@ def _get_randomize(database, table, columns, column, mapping_database, mapping_t column)) return ' '.join(query) + +# pylint: enable=too-many-arguments