Handle backups made with newer versions of mydumper (#10)

* Handle backups made with newer versions of mydumper

* Cleanup

* Fix test

* Hardcoded fix

* Improved tests

* Improved tests

* Fix failing tests

* More stict

* Cs fixes (#11)

* Cs fixes

* Cs fixes

* Fix requirements

* Cs fixes

* Cleanup

.travis.yml

@@ -3,16 +3,18 @@ language: python
 python:
   - "2.7"
   - "3.5"
+  - "3.6"
 
 services:
   - mysql
 
 before_install:
-  - mysql -uroot -e 'CREATE DATABASE IF NOT EXISTS `untraceables_test`;'
-  - mysql -uroot -e "CREATE USER 'untraceables'@'localhost' IDENTIFIED BY 'mmRXHqnc3zSshYjxSv8n';"
-  - mysql -uroot -e "GRANT ALL PRIVILEGES ON untraceables_test . * TO 'untraceables'@'localhost';"
-  - mysql -uroot -D untraceables_test -e 'CREATE TABLE `users` (`id` int(10) unsigned NOT NULL, `mapped_id` int(10) unsigned NOT NULL, PRIMARY KEY (`id`), UNIQUE KEY `mapped_id` (`mapped_id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8;';
-  - mysql -uroot -D untraceables_test -e 'INSERT INTO `users` (`id`, `mapped_id`) VALUES (0, 10), (1, 9), (2, 8);';
+  - |
+    mysql -uroot -e 'CREATE DATABASE IF NOT EXISTS `untraceables_test`;'
+    mysql -uroot -e "CREATE USER 'untraceables'@'localhost' IDENTIFIED BY 'mmRXHqnc3zSshYjxSv8n';"
+    mysql -uroot -e "GRANT ALL PRIVILEGES ON untraceables_test . * TO 'untraceables'@'localhost';"
+    mysql -uroot -D untraceables_test -e 'CREATE TABLE `users` (`id` int(10) unsigned NOT NULL, `mapped_id` int(10) unsigned NOT NULL, PRIMARY KEY (`id`), UNIQUE KEY `mapped_id` (`mapped_id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8;';
+    mysql -uroot -D untraceables_test -e 'INSERT INTO `users` (`id`, `mapped_id`) VALUES (0, 10), (1, 9), (2, 8);';
 
 install: make init-dev
 

Makefile

@@ -9,9 +9,15 @@ init-dev:
 source:
 	python setup.py sdist
 
-check:
-	find . -name \*.py | xargs pycodestyle --first
-	find bin -type f | xargs pycodestyle --first
+pycodestyle:
+	find . -type f -name \*.py | xargs --no-run-if-empty pycodestyle --first
+	find bin -type f | xargs --no-run-if-empty pycodestyle --first
+
+pylint:
+	find . -type f -name \*.py | xargs --no-run-if-empty pylint
+	find bin -type f | xargs --no-run-if-empty pylint
+
+check: pycodestyle pylint
 
 test:
 	nosetests -v

README.md

@@ -8,6 +8,7 @@
 
 * Python 2.7
 * Python 3.5
+* Python 3.6
 
 ## Usage
 

bin/randomize-ids

@@ -6,6 +6,8 @@
 Randomizes IDs for a given set of tables making them untraceable across environments
 """
 
+# pylint: disable=invalid-name
+
 from __future__ import print_function
 from __future__ import absolute_import
 import argparse

pylintrc

@@ -1,8 +1,20 @@
+[MASTER]
+extension-pkg-whitelist=MySQLdb
+
 [FORMAT]
 
 max-line-length=120
 
 indent-string='    '
 
-[TYPECHECK]
-ignored-classes=MySQLdb
+[SIMILARITIES]
+min-similarity-lines=4
+ignore-comments=yes
+ignore-docstrings=yes
+ignore-imports=yes
+
+[MESSAGES CONTROL]
+disable=broad-except,too-many-locals
+
+[BASIC]
+good-names=f,e

requirements-dev.txt

@@ -2,5 +2,6 @@ configobj==5.0.6
 mycli==1.20.1
 mysqlclient==1.4.6
 nose==1.3.7
-pycodestyle==2.5.0
-pylint==1.9.5
+pycodestyle==2.6.0
+pylint==2.6.2 ; python_version >= '3'
+pylint==1.9.5 ; python_version < '3'

setup.py

@@ -1,12 +1,21 @@
 # -*- coding: utf-8 -*-
 
+"""
+The setup script is the centre of all activity in building, distributing, and installing modules.
+"""
+
 from __future__ import absolute_import
 from setuptools import setup, find_packages
 
 
 def readme():
-    with open('README.md') as f:
-        return f.read()
+    """
+    Return README content.
+
+    :return: The specified number of bytes from the file
+    """
+    with open('README.md') as filepointer:
+        return filepointer.read()
 
 
 setup(name='untraceables',

untraceables/test/test_configuration.py

@@ -1,5 +1,9 @@
 # -*- coding: utf-8 -*-
 
+"""
+Tests for `configuration_utility`.
+"""
+
 from __future__ import absolute_import
 import os
 import unittest
@@ -8,6 +12,9 @@
 
 
 class TestConfiguration(unittest.TestCase):
+    """
+    TestCase.
+    """
 
     def test_read_file(self):
         """
@@ -29,7 +36,7 @@ def test_read_file(self):
         expected = 'dolor'
         self.assertEqual(expected, actual['main']['password'])
 
-    def test_read_file(self):
+    def test_read_file_0(self):
         """
         Tests `read_xclude_regexes_file`.
 
@@ -40,7 +47,7 @@ def test_read_file(self):
         actual = configuration_utility.read_xclude_regexes_file('lorem')
         self.assertEqual(expected, actual)
 
-    def test_read_file_0(self):
+    def test_read_file_1(self):
         """
         Tests `read_xclude_regexes_file`.
 
@@ -56,8 +63,9 @@ def test_read_file_0(self):
                     r'^users\.user_id$',
                     r'^users\..*user_id$']
         actual = configuration_utility.read_xclude_regexes_file(filename)
+        self.assertEqual(expected, actual)
 
-    def test_read_file_1(self):
+    def test_read_file_2(self):
         """
         Tests `read_xclude_regexes_file`.
 
@@ -67,6 +75,7 @@ def test_read_file_1(self):
         filename = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'include-from-1')
         expected = []
         actual = configuration_utility.read_xclude_regexes_file(filename)
+        self.assertEqual(expected, actual)
 
 
 suite = unittest.TestLoader().loadTestsFromTestCase(TestConfiguration)

untraceables/test/test_file.py

@@ -1,12 +1,20 @@
 # -*- coding: utf-8 -*-
 
+"""
+Tests for `file_utility`.
+"""
+
 from __future__ import absolute_import
 import os
 import unittest
+
 from untraceables.utilities import file as file_utility
 
 
 class TestFile(unittest.TestCase):
+    """
+    TestCase.
+    """
 
     def test_get_sorted_file_list(self):
         """
@@ -18,12 +26,12 @@ def test_get_sorted_file_list(self):
                     'test_split_file_0.sql', 'test_split_file_1.sql', 'test_split_file_2.sql',
                     'untraceables.cfg']
         actual = file_utility.get_sorted_file_list(path)
-        self.assertEquals(expected, list(actual))
+        self.assertEqual(expected, list(actual))
 
         path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data-non-existing')
         expected = []
         actual = file_utility.get_sorted_file_list(path)
-        self.assertEquals(expected, list(actual))
+        self.assertEqual(expected, list(actual))
 
 
 suite = unittest.TestLoader().loadTestsFromTestCase(TestFile)

untraceables/test/test_filter.py

@@ -1,11 +1,19 @@
 # -*- coding: utf-8 -*-
 
+"""
+Tests for `filter_utility`.
+"""
+
 from __future__ import absolute_import
 import unittest
+
 from untraceables.utilities import filter as filter_utility
 
 
 class TestFilter(unittest.TestCase):
+    """
+    TestCase.
+    """
 
     def test_show_tables(self):
         """
@@ -16,34 +24,34 @@ def test_show_tables(self):
         inclusive_regexes = exclusive_regexes = []
         expected = set([])
         actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes)
-        self.assertEquals(expected, actual)
+        self.assertEqual(expected, actual)
 
         table_columns = iter(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing'])
         inclusive_regexes = exclusive_regexes = []
         expected = set([])
         actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes)
-        self.assertEquals(expected, actual)
+        self.assertEqual(expected, actual)
 
         table_columns = iter(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing'])
         inclusive_regexes = [r'^.*\..*$']
         exclusive_regexes = []
         expected = set(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing'])
         actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes)
-        self.assertEquals(expected, actual)
+        self.assertEqual(expected, actual)
 
         table_columns = iter(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing'])
         inclusive_regexes = [r'^.*\..*$']
         exclusive_regexes = [r'^sit\..*$']
         expected = set(['ipsum.dolor', 'consectetur.adipiscing'])
         actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes)
-        self.assertEquals(expected, actual)
+        self.assertEqual(expected, actual)
 
         table_columns = iter(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing'])
         inclusive_regexes = [r'^.*\..*$']
         exclusive_regexes = [r'^.*\.adipiscing$']
         expected = set(['ipsum.dolor', 'sit.amet'])
         actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes)
-        self.assertEquals(expected, actual)
+        self.assertEqual(expected, actual)
 
     def test_table_names_from_mydumper_backup(self):
         """
@@ -55,22 +63,27 @@ def test_table_names_from_mydumper_backup(self):
         expected = []
         actual = filter_utility.table_names_from_mydumper_backup(files, suffixed_database)
         self.assertTrue(hasattr(actual, 'next') or hasattr(actual, '__next__'))
-        self.assertEquals(expected, list(actual))
+        self.assertEqual(expected, list(actual))
 
         files = ['ipsum.dolor-schema.sql', 'ipsum.dolor.sql',
-                 'ipsum.consectetur-schema.sql', 'ipsum.consectetur.sql']
+                 'ipsum.consectetur-schema.sql', 'ipsum.consectetur.sql',
+                 'ipsum-schema-create.sql', 'ipsum.sit-schema.sql', 'ipsum.sit.00000.sql',
+                 'ipsum.sit.00001.sql', 'ipsum.sit.12356.sql', 'ipsum.sit.99999.sql']
         suffixed_database = 'ipsum.'
-        expected = ['ipsum.dolor.sql', 'ipsum.consectetur.sql']
+        expected = ['ipsum.dolor.sql', 'ipsum.consectetur.sql', 'ipsum.sit.00000.sql',
+                    'ipsum.sit.00001.sql', 'ipsum.sit.12356.sql', 'ipsum.sit.99999.sql']
         actual = filter_utility.table_names_from_mydumper_backup(files, suffixed_database)
         self.assertTrue(hasattr(actual, 'next') or hasattr(actual, '__next__'))
-        self.assertEquals(expected, list(actual))
+        self.assertEqual(expected, list(actual))
 
         files = ['ipsum.dolor-schema.sql', 'ipsum.dolor.sql',
-                 'consectetur.adipiscing-schema.sql', 'consectetur.adipiscing.sql']
+                 'ipsum-schema-create.sql', 'ipsum.sit-schema.sql', 'ipsum.sit.00000.sql',
+                 'consectetur.adipiscing-schema.sql', 'consectetur.adipiscing.sql',
+                 'elit-schema-create.sql', 'elit.mollis-schema.sql', 'elit.mollis.00000.sql']
         suffixed_database = 'ipsum.'
-        expected = ['ipsum.dolor.sql']
+        expected = ['ipsum.dolor.sql', 'ipsum.sit.00000.sql']
         actual = filter_utility.table_names_from_mydumper_backup(files, suffixed_database)
-        self.assertEquals(expected, list(actual))
+        self.assertEqual(expected, list(actual))
 
 
 suite = unittest.TestLoader().loadTestsFromTestCase(TestFilter)