From 2b74f37d944de5ebfbad1a1ffae4e875e1266c2b Mon Sep 17 00:00:00 2001 From: Mischa ter Smitten Date: Mon, 30 Mar 2020 12:12:07 +0200 Subject: [PATCH] Make python-untraceables python 3 compatible (#7) --- .travis.yml | 5 +- README.md | 1 + bin/randomize-ids | 456 ++++++++++++------------ pylintrc | 2 +- requirements-dev.txt | 9 +- requirements.txt | 2 +- setup.py | 5 +- untraceables/test/test_configuration.py | 117 +++--- untraceables/test/test_file.py | 34 +- untraceables/test/test_filter.py | 130 +++---- untraceables/test/test_formatter.py | 180 +++++----- untraceables/test/test_mysql.py | 435 +++++++++++----------- untraceables/test/test_query.py | 208 +++++------ untraceables/test/test_validation.py | 58 +-- untraceables/utilities/cli.py | 33 +- untraceables/utilities/configuration.py | 70 ++-- untraceables/utilities/file.py | 23 +- untraceables/utilities/filter.py | 91 ++--- untraceables/utilities/formatter.py | 145 ++++---- untraceables/utilities/mysql.py | 217 ++++++----- untraceables/utilities/query.py | 237 ++++++------ untraceables/utilities/validation.py | 20 +- 22 files changed, 1241 insertions(+), 1237 deletions(-) diff --git a/.travis.yml b/.travis.yml index 01c68a9..93155d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python python: - "2.7" + - "3.5" services: - mysql @@ -26,6 +27,6 @@ script: notifications: email: false - hipchat: + slack: rooms: - secure: X5mHSHzsNBgQNKnwIFuSzagSHUqyPLGWL1SW6aMHRd4WtdAEtfN3w+iyvT1s0v7F4p9xZc3PEzV8WbXtsELkDEnYvLyucS8MVMQJ1ohuDig32t4TIwMnEZF0IyhDTlUtG31wnLWxxM0Kmx+/6xrg7I9tx2EA0w2opof2K1cHBVARFEq/sFBF/bOthLfR9dM4paAkELZMgg/CsK4T9S9SWo4NsvwUbP3bWV3Gh6dYF3g5nd7DhqxUvusL9oMu5JXA5U4H7dCB7aQRAjg12Fdg/F/ZJelVT+Deg3zGHep8zTEr4ykVWpoVko6VK9+Jh8WSnazG7XCEMNNatmR48CTXuoj5axLTjP0yyfeniauIFwapO3jaf+Vn/HOAa3xNQz/MBg24J9P74CCeYDJ18uu5WqcNetpw/re4N6edPGWBfbk332JoMvPU47RlX24n6rxghO62zA4wf/uohmeEdqxlDIfeofPeCKBuxeMT/wAVKxFB75fmBlrAQcGQuVgvfRLOEUb1ONTFsl4RaTSIRsxFivRCWp5jUeUp4BxYYKpFKhBawIvaFQtriy/PbTZUBlBmOATGmpAL/bgmfuEWKTZ8uVJYkj/VoI36C6hqWbzJKuxzWn7tFeeQC2dcSpdhZx7aacdhEGYlv00v9X+AIXSGxNhQIIraGkEsvqqj2F2kag4= + secure: "4/IX3mVZcuDbPszD3zRgcu3FLWQafgaWdj7jZ+J9xdt94j0rtlanw3cCifed8+RXtarIGQUk1mUt6UXF9wdG3lgaS3zCLAXJfkyBs/i8gKOUQRWpDATpFCkPGOyz2FcYKLAdefkMQz8SmrJ5aLVb20jqxYAsgs8SNsrf3z7LrAeOY8tCPgBVXSQjIGfU8TckEObDLuIdZcPrVgGRoHcT2kBMQk4dlcosChlItv7pi/dB6svJM89Nrv1QRd644NNRwslQ2+jw2CPitbHbRrEWjp0NRg18e9io0P7iMOnrRjuWPhY6+3Nzxv7/q9IO2rK4zXxGK4rlJNMC/61qYb1Khak0eeLhUaatsQaIpPTXG30SEYvIyZpNJQGpawN3dyBTQW17D4kBni/N9ruXs+rhYve/k7WS2I1BjrDellHmIJ4PT/Y5pOAx5HE00w1f00K3LZLaBizhU/R7WzRgbhQjqO2RyItZZGG0fX0lcyUVRP5RPFJAPDX5M8vYyfT/HWLB80q1PSY8Zt+x0vVOtkzqmQkfBnr1hx9XYlT9zbkBs1tw8UxsDYQC2f+aXFKtq0EHBDijCWy05VEGIR+WwID9MHBLrd+EYec713DVrk7TFwI34t3izfhSaU0novOdZqpliqKepBnF92FNKRj5nQ1UEKjDgqat2bm26MhR43+tnH0=" diff --git a/README.md b/README.md index 4a63517..7a4a56b 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ ## Requirements * Python 2.7 +* Python 3.5 ## Usage diff --git a/bin/randomize-ids b/bin/randomize-ids index c46712f..a85f4fd 100755 --- a/bin/randomize-ids +++ b/bin/randomize-ids @@ -7,13 +7,12 @@ Randomizes IDs for a given set of tables making them untraceable across environm """ from __future__ import print_function - +from __future__ import absolute_import import argparse import re import sys import os from itertools import chain - import untraceables from untraceables.utilities import cli as cli_utility from untraceables.utilities import configuration as configuration_utility @@ -26,279 +25,280 @@ from untraceables.utilities import validation as validation_utility def get_include_from_mydumper_backup(args): - """ - Wrapper for `get-include-from-mydumper-backup` subcommand. + """ + Wrapper for `get-include-from-mydumper-backup` subcommand. - :type args: argparse.Namespace - :param args: Arguments - """ + :type args: argparse.Namespace + :param args: Arguments + """ - config = configuration_utility.read_file(cli_utility.CONFIGURATION_FILE) - if config is False: - cli_utility.print_e('Could not find or read configuration file') + config = configuration_utility.read_file(cli_utility.CONFIGURATION_FILE) + if config is False: + cli_utility.print_e('Could not find or read configuration file') - database = args.database - path = args.path - inclusive_regexes = args.include_regex + database = args.database + path = args.path + inclusive_regexes = args.include_regex - database_table_delimiter = r'\.' - suffixed_database = database + '.' + database_table_delimiter = r'\.' + suffixed_database = database + '.' - files = file_utility.get_sorted_file_list(path) - filtered_files = filter_utility.table_names_from_mydumper_backup(files, suffixed_database) - formatted_files = formatter_utility.table_names_from_mydumper_backup(filtered_files, suffixed_database) + files = file_utility.get_sorted_file_list(path) + filtered_files = filter_utility.table_names_from_mydumper_backup(files, suffixed_database) + formatted_files = formatter_utility.table_names_from_mydumper_backup(filtered_files, suffixed_database) - for file_basename in formatted_files: - for inclusive_regex in inclusive_regexes: - table_regex, field_regex = formatter_utility.inclusive_regex_in(inclusive_regex, database_table_delimiter) - if re.match(re.compile(table_regex), file_basename): - print(formatter_utility.inclusive_regex_out(file_basename, field_regex, database_table_delimiter)) + for file_basename in formatted_files: + for inclusive_regex in inclusive_regexes: + table_regex, field_regex = formatter_utility.inclusive_regex_in(inclusive_regex, database_table_delimiter) + if re.match(re.compile(table_regex), file_basename): + print(formatter_utility.inclusive_regex_out(file_basename, field_regex, database_table_delimiter)) def get_table_list(args): - """ - Wrapper for `get-table-list` subcommand. + """ + Wrapper for `get-table-list` subcommand. - :type args: argparse.Namespace - :param args: Arguments - """ + :type args: argparse.Namespace + :param args: Arguments + """ - config = configuration_utility.read_file(cli_utility.CONFIGURATION_FILE) - if config is False: - cli_utility.print_e('Could not find or read configuration file') + config = configuration_utility.read_file(cli_utility.CONFIGURATION_FILE) + if config is False: + cli_utility.print_e('Could not find or read configuration file') - host, user, password = cli_utility.config_unpack(config) + host, user, password = cli_utility.config_unpack(config) - database = args.database - inclusive_regexes = args.include_regex - if not inclusive_regexes: - inclusive_regexes = configuration_utility.read_xclude_regexes_file(args.include_from) - exclusive_regexes = args.exclude_regex - if not inclusive_regexes: - exclusive_regexes = configuration_utility.read_xclude_regexes_file(args.exclude_from) + database = args.database + inclusive_regexes = args.include_regex + if not inclusive_regexes: + inclusive_regexes = configuration_utility.read_xclude_regexes_file(args.include_from) + exclusive_regexes = args.exclude_regex + if not inclusive_regexes: + exclusive_regexes = configuration_utility.read_xclude_regexes_file(args.exclude_from) - connection = cursor = None - try: - connection = mysql_utility.get_connection(host, user, password, database) - cursor = mysql_utility.get_cursor(connection) + connection = cursor = None + try: + connection = mysql_utility.get_connection(host, user, password, database) + cursor = mysql_utility.get_cursor(connection) - show_tables = mysql_utility.get_show_tables(cursor, database) - formatted_show_tables = formatter_utility.show_tables(show_tables) + show_tables = mysql_utility.get_show_tables(cursor, database) + formatted_show_tables = formatter_utility.show_tables(show_tables) - filtered_columns = filter_utility.show_tables(formatted_show_tables, inclusive_regexes, exclusive_regexes) - formatted_columns = formatter_utility.table_columns_tsv(database, sorted(filtered_columns)) + filtered_columns = filter_utility.show_tables(formatted_show_tables, inclusive_regexes, exclusive_regexes) + formatted_columns = formatter_utility.table_columns_tsv(database, sorted(filtered_columns)) - print(*formatted_columns, sep=os.linesep) - except Exception, e: - cli_utility.print_e(e) - finally: - mysql_utility.close_connection_and_cursor(connection, cursor) + print(*formatted_columns, sep=os.linesep) + except Exception as e: + cli_utility.print_e(e) + finally: + mysql_utility.close_connection_and_cursor(connection, cursor) def get_sql(args): - """ - Wrapper for `get-sql` subcommand. + """ + Wrapper for `get-sql` subcommand. - :type args: argparse.Namespace - :param args: Arguments - """ + :type args: argparse.Namespace + :param args: Arguments + """ - config = configuration_utility.read_file(cli_utility.CONFIGURATION_FILE) - if config is False: - cli_utility.print_e('Could not find or read configuration file') + config = configuration_utility.read_file(cli_utility.CONFIGURATION_FILE) + if config is False: + cli_utility.print_e('Could not find or read configuration file') - host, user, password = cli_utility.config_unpack(config) + host, user, password = cli_utility.config_unpack(config) - database = args.database - table = args.table - column = args.column - mapping_database = args.mapping_database or untraceables.__name__ - mapping_table = args.mapping_table or table + database = args.database + table = args.table + column = args.column + mapping_database = args.mapping_database or untraceables.__name__ + mapping_table = args.mapping_table or table - connection = cursor = None - try: - connection = mysql_utility.get_connection(host, user, password, database) - cursor = mysql_utility.get_cursor(connection) + connection = cursor = None + try: + connection = mysql_utility.get_connection(host, user, password, database) + cursor = mysql_utility.get_cursor(connection) - columns = mysql_utility.get_show_columns(cursor, table) + columns = mysql_utility.get_show_columns(cursor, table) - max_id = mysql_utility.get_max_id(cursor, database, table, column) - mapping_max_id = mysql_utility.get_max_id(cursor, mapping_database, mapping_table, untraceables.MAPPING_ID_FIELD) + max_id = mysql_utility.get_max_id(cursor, database, table, column) + mapping_max_id = mysql_utility.get_max_id(cursor, mapping_database, mapping_table, + untraceables.MAPPING_ID_FIELD) - if validation_utility.check_max_ids(max_id, mapping_max_id): - cli_utility.print_e('Could not randomize IDs because the maximum ID of the mapping table is not sufficient') + if validation_utility.check_max_ids(max_id, mapping_max_id): + cli_utility.print_e('Could not randomize IDs because the maximum ID of the mapping table is not sufficient') - randomize_ids_queries = query_utility.get_randomize(database, table, columns, column, - mapping_database, mapping_table) - formatted_randomize_ids_queries = formatter_utility.randomize_queries(randomize_ids_queries) + randomize_ids_queries = query_utility.get_randomize(database, table, columns, column, + mapping_database, mapping_table) + formatted_randomize_ids_queries = formatter_utility.randomize_queries(randomize_ids_queries) - print(formatted_randomize_ids_queries) - except Exception, e: - cli_utility.print_e(e) - finally: - mysql_utility.close_connection_and_cursor(connection, cursor) + print(formatted_randomize_ids_queries) + except Exception as e: + cli_utility.print_e(e) + finally: + mysql_utility.close_connection_and_cursor(connection, cursor) def run_sql(args): - """ - Wrapper for `run-sql` subcommand. + """ + Wrapper for `run-sql` subcommand. - :type args: argparse.Namespace - :param args: Arguments - """ + :type args: argparse.Namespace + :param args: Arguments + """ - config = configuration_utility.read_file(cli_utility.CONFIGURATION_FILE) - if config is False: - cli_utility.print_e('Could not find or read configuration file') + config = configuration_utility.read_file(cli_utility.CONFIGURATION_FILE) + if config is False: + cli_utility.print_e('Could not find or read configuration file') - host, user, password = cli_utility.config_unpack(config) + host, user, password = cli_utility.config_unpack(config) - database = args.database - foreign_key_checks = args.foreign_key_checks - unique_checks = args.unique_checks + database = args.database + foreign_key_checks = args.foreign_key_checks + unique_checks = args.unique_checks - if sys.stdin.isatty(): - cli_utility.print_e('Could not read any data from stdin') + if sys.stdin.isatty(): + cli_utility.print_e('Could not read any data from stdin') - foreign_key_checks_off = [] - foreign_key_checks_on = [] - if not foreign_key_checks: - foreign_key_checks_off = [query_utility.get_foreign_key_checks(foreign_key_checks)] - foreign_key_checks_on = [query_utility.get_foreign_key_checks(not foreign_key_checks)] + foreign_key_checks_off = [] + foreign_key_checks_on = [] + if not foreign_key_checks: + foreign_key_checks_off = [query_utility.get_foreign_key_checks(foreign_key_checks)] + foreign_key_checks_on = [query_utility.get_foreign_key_checks(not foreign_key_checks)] - unique_checks_off = [] - unique_checks_on = [] - if not foreign_key_checks: - unique_checks_off = [query_utility.get_unique_checks(unique_checks)] - unique_checks_on = [query_utility.get_unique_checks(not unique_checks)] + unique_checks_off = [] + unique_checks_on = [] + if not foreign_key_checks: + unique_checks_off = [query_utility.get_unique_checks(unique_checks)] + unique_checks_on = [query_utility.get_unique_checks(not unique_checks)] - statements_from_stdin = mysql_utility.split_file(sys.stdin) + statements_from_stdin = mysql_utility.split_file(sys.stdin) - connection = cursor = None - try: - connection = mysql_utility.get_connection(host, user, password, database) - connection.autocommit(True) - cursor = mysql_utility.get_cursor(connection) + connection = cursor = None + try: + connection = mysql_utility.get_connection(host, user, password, database) + connection.autocommit(True) + cursor = mysql_utility.get_cursor(connection) - statements = chain(iter(foreign_key_checks_off), iter(unique_checks_off), - statements_from_stdin, - iter(unique_checks_on), iter(foreign_key_checks_on)) - for statement in statements: - stripped_statement = statement.strip() - if stripped_statement != '': - cursor.execute(stripped_statement) + statements = chain(iter(foreign_key_checks_off), iter(unique_checks_off), + statements_from_stdin, + iter(unique_checks_on), iter(foreign_key_checks_on)) + for statement in statements: + stripped_statement = statement.strip() + if stripped_statement != '': + cursor.execute(stripped_statement) - except Exception, e: - cli_utility.print_e(e) - finally: - mysql_utility.close_connection_and_cursor(connection, cursor) + except Exception as e: + cli_utility.print_e(e) + finally: + mysql_utility.close_connection_and_cursor(connection, cursor) def main(): - """ - Main. - """ - - parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers(help='Commands') - - # The "get-include-from-mydumper-backup" command - include_file_from_mydumper_backup_parser = subparsers.add_parser('get-include-from-mydumper-backup', - help='Generates a list of tables to be randomized') - include_file_from_mydumper_backup_parser.set_defaults(func=get_include_from_mydumper_backup) - include_file_from_mydumper_backup_parser.add_argument('-d', '--database', - type=str, - required=True, - help='A database name (e.g. `example_com_www`)') - include_file_from_mydumper_backup_parser.add_argument('-p', '--path', - type=str, - required=True, - help='A mydumper backup path (e.g. `/data/backup/mydumper`)') - include_file_from_mydumper_backup_parser.add_argument('-i', '--include-regex', - type=str, - default=[], - action='append', - help=('A regex for `table.column` inclusion ' - r'''(e.g. `'^users\.id$'`)''')) - - # The "get-table-list" command - get_table_list_parser = subparsers.add_parser('get-table-list', help='Generates a list of tables to be randomized') - get_table_list_parser.set_defaults(func=get_table_list) - get_table_list_parser.add_argument('-d', '--database', - type=str, - required=True, - help='A database name (e.g. `example_com_www`)') - get_table_list_parser_include_group = get_table_list_parser.add_mutually_exclusive_group() - get_table_list_parser_include_group.add_argument('-i', '--include-regex', - type=str, - default=[], - action='append', - help=('A regex for `table.column` inclusion ' - r'''(e.g. `'^users\.id$'`)''')) - get_table_list_parser_include_group.add_argument('--include-from', - type=str, - default='', - help='A file (with (regexes) for `table.column` inclusion') - get_table_list_parser_exclude_group = get_table_list_parser.add_mutually_exclusive_group() - get_table_list_parser_exclude_group.add_argument('-e', '--exclude-regex', - type=str, - default=[], - action='append', - help=('A regex for `table.column` exclusion ' - r'''(e.g. `'^user_application_typesucces_properties\.''' - '''typesucces_user_id$'`)''')) - get_table_list_parser_exclude_group.add_argument('--exclude-from', - type=str, - default='', - help='A file (with regexes) for `table.column` exclusion') - - # The "get-sql" command - get_sql_parser = subparsers.add_parser('get-sql', help='Generates sql to randomize id\'s of a given table') - get_sql_parser.set_defaults(func=get_sql) - get_sql_parser.add_argument('-d', '--database', - type=str, - required=True, - help='A database name (e.g. `example_com_www`)') - get_sql_parser.add_argument('-t', '--table', type=str, required=True, help='A table name (e.g. `users`)') - get_sql_parser.add_argument('-c', '--column', type=str, required=True, help='A column name (e.g. `id`)') - get_sql_parser.add_argument('--mapping-database', - type=str, - help='A (mapping) database name (e.g. `untraceables`)') - get_sql_parser.add_argument('--mapping-table', type=str, help='A (mapping) table name (e.g. `users`)') - - # The "run-sql" command - run_sql_parser = subparsers.add_parser('run-sql', help='Runs sql from stdin') - run_sql_parser.set_defaults(func=run_sql) - run_sql_parser.add_argument('-d', '--database', - type=str, - required=True, - help='A database name (e.g. `example_com_www`)') - run_sql_parser.add_argument('--foreign-key-checks', - action='store_true', - dest='foreign_key_checks', - help='Whether or not to enable FOREIGN_KEY_CHECKS') - run_sql_parser.add_argument('--no-foreign-key-checks', - action='store_false', - dest='foreign_key_checks', - help='Whether or not to enable FOREIGN_KEY_CHECKS') - run_sql_parser.set_defaults(foreign_key_checks=True) - run_sql_parser.add_argument('--unique-checks', - action='store_true', - dest='unique_checks', - help='Whether or not to enable UNIQUE_CHECKS') - run_sql_parser.add_argument('--no-unique-checks', - action='store_false', - dest='unique_checks', - help='Whether or not to enable UNIQUE_CHECKS') - run_sql_parser.set_defaults(foreign_key_checks=True) - - parser.add_argument('-v', '--verbose', action='store_true', help='Be more verbose') - - args = parser.parse_args() - args.func(args) - - sys.exit(0) + """ + Main. + """ + + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(help='Commands') + + # The "get-include-from-mydumper-backup" command + include_file_from_mydumper_backup_parser = subparsers.add_parser('get-include-from-mydumper-backup', + help='Generates a list of tables to be randomized') + include_file_from_mydumper_backup_parser.set_defaults(func=get_include_from_mydumper_backup) + include_file_from_mydumper_backup_parser.add_argument('-d', '--database', + type=str, + required=True, + help='A database name (e.g. `example_com_www`)') + include_file_from_mydumper_backup_parser.add_argument('-p', '--path', + type=str, + required=True, + help='A mydumper backup path (e.g. `/data/backup/mydumper`)') + include_file_from_mydumper_backup_parser.add_argument('-i', '--include-regex', + type=str, + default=[], + action='append', + help=('A regex for `table.column` inclusion ' + r'''(e.g. `'^users\.id$'`)''')) + + # The "get-table-list" command + get_table_list_parser = subparsers.add_parser('get-table-list', help='Generates a list of tables to be randomized') + get_table_list_parser.set_defaults(func=get_table_list) + get_table_list_parser.add_argument('-d', '--database', + type=str, + required=True, + help='A database name (e.g. `example_com_www`)') + get_table_list_parser_include_group = get_table_list_parser.add_mutually_exclusive_group() + get_table_list_parser_include_group.add_argument('-i', '--include-regex', + type=str, + default=[], + action='append', + help=('A regex for `table.column` inclusion ' + r'''(e.g. `'^users\.id$'`)''')) + get_table_list_parser_include_group.add_argument('--include-from', + type=str, + default='', + help='A file (with (regexes) for `table.column` inclusion') + get_table_list_parser_exclude_group = get_table_list_parser.add_mutually_exclusive_group() + get_table_list_parser_exclude_group.add_argument('-e', '--exclude-regex', + type=str, + default=[], + action='append', + help=('A regex for `table.column` exclusion ' + r'''(e.g. `'^user_application_typesucces_properties\.''' + '''typesucces_user_id$'`)''')) + get_table_list_parser_exclude_group.add_argument('--exclude-from', + type=str, + default='', + help='A file (with regexes) for `table.column` exclusion') + + # The "get-sql" command + get_sql_parser = subparsers.add_parser('get-sql', help='Generates sql to randomize id\'s of a given table') + get_sql_parser.set_defaults(func=get_sql) + get_sql_parser.add_argument('-d', '--database', + type=str, + required=True, + help='A database name (e.g. `example_com_www`)') + get_sql_parser.add_argument('-t', '--table', type=str, required=True, help='A table name (e.g. `users`)') + get_sql_parser.add_argument('-c', '--column', type=str, required=True, help='A column name (e.g. `id`)') + get_sql_parser.add_argument('--mapping-database', + type=str, + help='A (mapping) database name (e.g. `untraceables`)') + get_sql_parser.add_argument('--mapping-table', type=str, help='A (mapping) table name (e.g. `users`)') + + # The "run-sql" command + run_sql_parser = subparsers.add_parser('run-sql', help='Runs sql from stdin') + run_sql_parser.set_defaults(func=run_sql) + run_sql_parser.add_argument('-d', '--database', + type=str, + required=True, + help='A database name (e.g. `example_com_www`)') + run_sql_parser.add_argument('--foreign-key-checks', + action='store_true', + dest='foreign_key_checks', + help='Whether or not to enable FOREIGN_KEY_CHECKS') + run_sql_parser.add_argument('--no-foreign-key-checks', + action='store_false', + dest='foreign_key_checks', + help='Whether or not to enable FOREIGN_KEY_CHECKS') + run_sql_parser.set_defaults(foreign_key_checks=True) + run_sql_parser.add_argument('--unique-checks', + action='store_true', + dest='unique_checks', + help='Whether or not to enable UNIQUE_CHECKS') + run_sql_parser.add_argument('--no-unique-checks', + action='store_false', + dest='unique_checks', + help='Whether or not to enable UNIQUE_CHECKS') + run_sql_parser.set_defaults(foreign_key_checks=True) + + parser.add_argument('-v', '--verbose', action='store_true', help='Be more verbose') + + args = parser.parse_args() + args.func(args) + + sys.exit(0) if __name__ == '__main__': - main() + main() diff --git a/pylintrc b/pylintrc index be51cba..9c93cce 100644 --- a/pylintrc +++ b/pylintrc @@ -2,7 +2,7 @@ max-line-length=120 -indent-string=' ' +indent-string=' ' [TYPECHECK] ignored-classes=MySQLdb diff --git a/requirements-dev.txt b/requirements-dev.txt index c591342..04aa2e9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,6 @@ configobj==5.0.6 -mycli==1.18.2 -MySQL-python==1.2.5 +mycli==1.20.1 +mysqlclient==1.4.6 nose==1.3.7 -pep8==1.7.1 -pycodestyle==2.4.0 -pylint==1.9.3 +pycodestyle==2.5.0 +pylint==1.9.5 diff --git a/requirements.txt b/requirements.txt index bfb8bdd..7915eba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ configobj==5.0.6 -MySQL-python==1.2.5 +mysqlclient==1.4.6 diff --git a/setup.py b/setup.py index b557f41..5da6d68 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,12 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import from setuptools import setup, find_packages def readme(): - with open('README.md') as f: - return f.read() + with open('README.md') as f: + return f.read() setup(name='untraceables', diff --git a/untraceables/test/test_configuration.py b/untraceables/test/test_configuration.py index 7db5aca..5df3bba 100644 --- a/untraceables/test/test_configuration.py +++ b/untraceables/test/test_configuration.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import import os import unittest @@ -8,64 +9,64 @@ class TestConfiguration(unittest.TestCase): - def test_read_file(self): - """ - Tests `read_file`. - """ - - filename = 'lorem' - actual = configuration_utility.read_file(filename) - self.assertFalse(actual) - - filename = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'untraceables.cfg') - actual = configuration_utility.read_file(filename) - expected = 'configobj.ConfigObj' - self.assertTrue(expected in str(type(actual))) - expected = 'lorem' - self.assertEqual(expected, actual['main']['host']) - expected = 'ipsum' - self.assertEqual(expected, actual['main']['user']) - expected = 'dolor' - self.assertEqual(expected, actual['main']['password']) - - def test_read_file(self): - """ - Tests `read_xclude_regexes_file`. - - Non-existing file. - """ - - expected = [] - actual = configuration_utility.read_xclude_regexes_file('lorem') - self.assertEqual(expected, actual) - - def test_read_file_0(self): - """ - Tests `read_xclude_regexes_file`. - - Non-empty file. - """ - - filename = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'include-from-0') - expected = [r'^audit_trails\.user_id$', - r'^audit_trails\..*user_id$', - r'^tickets\.user_id$', - r'^tickets\..*user_id$', - r'^users\.id$', - r'^users\.user_id$', - r'^users\..*user_id$'] - actual = configuration_utility.read_xclude_regexes_file(filename) - - def test_read_file_1(self): - """ - Tests `read_xclude_regexes_file`. - - Empty file. - """ - - filename = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'include-from-1') - expected = [] - actual = configuration_utility.read_xclude_regexes_file(filename) + def test_read_file(self): + """ + Tests `read_file`. + """ + + filename = 'lorem' + actual = configuration_utility.read_file(filename) + self.assertFalse(actual) + + filename = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'untraceables.cfg') + actual = configuration_utility.read_file(filename) + expected = 'configobj.ConfigObj' + self.assertTrue(expected in str(type(actual))) + expected = 'lorem' + self.assertEqual(expected, actual['main']['host']) + expected = 'ipsum' + self.assertEqual(expected, actual['main']['user']) + expected = 'dolor' + self.assertEqual(expected, actual['main']['password']) + + def test_read_file(self): + """ + Tests `read_xclude_regexes_file`. + + Non-existing file. + """ + + expected = [] + actual = configuration_utility.read_xclude_regexes_file('lorem') + self.assertEqual(expected, actual) + + def test_read_file_0(self): + """ + Tests `read_xclude_regexes_file`. + + Non-empty file. + """ + + filename = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'include-from-0') + expected = [r'^audit_trails\.user_id$', + r'^audit_trails\..*user_id$', + r'^tickets\.user_id$', + r'^tickets\..*user_id$', + r'^users\.id$', + r'^users\.user_id$', + r'^users\..*user_id$'] + actual = configuration_utility.read_xclude_regexes_file(filename) + + def test_read_file_1(self): + """ + Tests `read_xclude_regexes_file`. + + Empty file. + """ + + filename = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'include-from-1') + expected = [] + actual = configuration_utility.read_xclude_regexes_file(filename) suite = unittest.TestLoader().loadTestsFromTestCase(TestConfiguration) diff --git a/untraceables/test/test_file.py b/untraceables/test/test_file.py index 20eeaff..06304e5 100644 --- a/untraceables/test/test_file.py +++ b/untraceables/test/test_file.py @@ -1,29 +1,29 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import import os import unittest - from untraceables.utilities import file as file_utility class TestFile(unittest.TestCase): - def test_get_sorted_file_list(self): - """ - Tests `get_sorted_file_list`. - """ - - path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') - expected = ['include-from-0', 'include-from-1', - 'test_split_file_0.sql', 'test_split_file_1.sql', 'test_split_file_2.sql', - 'untraceables.cfg'] - actual = file_utility.get_sorted_file_list(path) - self.assertEquals(expected, list(actual)) - - path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data-non-existing') - expected = [] - actual = file_utility.get_sorted_file_list(path) - self.assertEquals(expected, list(actual)) + def test_get_sorted_file_list(self): + """ + Tests `get_sorted_file_list`. + """ + + path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') + expected = ['include-from-0', 'include-from-1', + 'test_split_file_0.sql', 'test_split_file_1.sql', 'test_split_file_2.sql', + 'untraceables.cfg'] + actual = file_utility.get_sorted_file_list(path) + self.assertEquals(expected, list(actual)) + + path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data-non-existing') + expected = [] + actual = file_utility.get_sorted_file_list(path) + self.assertEquals(expected, list(actual)) suite = unittest.TestLoader().loadTestsFromTestCase(TestFile) diff --git a/untraceables/test/test_filter.py b/untraceables/test/test_filter.py index 36ee53c..5a47687 100644 --- a/untraceables/test/test_filter.py +++ b/untraceables/test/test_filter.py @@ -1,76 +1,76 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import import unittest - from untraceables.utilities import filter as filter_utility class TestFilter(unittest.TestCase): - def test_show_tables(self): - """ - Tests `show_tables`. - """ - - table_columns = iter([]) - inclusive_regexes = exclusive_regexes = [] - expected = set([]) - actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes) - self.assertEquals(expected, actual) - - table_columns = iter(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing']) - inclusive_regexes = exclusive_regexes = [] - expected = set([]) - actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes) - self.assertEquals(expected, actual) - - table_columns = iter(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing']) - inclusive_regexes = [r'^.*\..*$'] - exclusive_regexes = [] - expected = set(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing']) - actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes) - self.assertEquals(expected, actual) - - table_columns = iter(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing']) - inclusive_regexes = [r'^.*\..*$'] - exclusive_regexes = [r'^sit\..*$'] - expected = set(['ipsum.dolor', 'consectetur.adipiscing']) - actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes) - self.assertEquals(expected, actual) - - table_columns = iter(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing']) - inclusive_regexes = [r'^.*\..*$'] - exclusive_regexes = [r'^.*\.adipiscing$'] - expected = set(['ipsum.dolor', 'sit.amet']) - actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes) - self.assertEquals(expected, actual) - - def test_table_names_from_mydumper_backup(self): - """ - Tests `table_names_from_mydumper_backup`. - """ - - files = [] - suffixed_database = 'ipsum.' - expected = [] - actual = filter_utility.table_names_from_mydumper_backup(files, suffixed_database) - self.assertTrue(hasattr(actual, 'next')) - self.assertEquals(expected, list(actual)) - - files = ['ipsum.dolor-schema.sql', 'ipsum.dolor.sql', - 'ipsum.consectetur-schema.sql', 'ipsum.consectetur.sql'] - suffixed_database = 'ipsum.' - expected = ['ipsum.dolor.sql', 'ipsum.consectetur.sql'] - actual = filter_utility.table_names_from_mydumper_backup(files, suffixed_database) - self.assertTrue(hasattr(actual, 'next')) - self.assertEquals(expected, list(actual)) - - files = ['ipsum.dolor-schema.sql', 'ipsum.dolor.sql', - 'consectetur.adipiscing-schema.sql', 'consectetur.adipiscing.sql'] - suffixed_database = 'ipsum.' - expected = ['ipsum.dolor.sql'] - actual = filter_utility.table_names_from_mydumper_backup(files, suffixed_database) - self.assertEquals(expected, list(actual)) + def test_show_tables(self): + """ + Tests `show_tables`. + """ + + table_columns = iter([]) + inclusive_regexes = exclusive_regexes = [] + expected = set([]) + actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes) + self.assertEquals(expected, actual) + + table_columns = iter(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing']) + inclusive_regexes = exclusive_regexes = [] + expected = set([]) + actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes) + self.assertEquals(expected, actual) + + table_columns = iter(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing']) + inclusive_regexes = [r'^.*\..*$'] + exclusive_regexes = [] + expected = set(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing']) + actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes) + self.assertEquals(expected, actual) + + table_columns = iter(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing']) + inclusive_regexes = [r'^.*\..*$'] + exclusive_regexes = [r'^sit\..*$'] + expected = set(['ipsum.dolor', 'consectetur.adipiscing']) + actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes) + self.assertEquals(expected, actual) + + table_columns = iter(['ipsum.dolor', 'sit.amet', 'consectetur.adipiscing']) + inclusive_regexes = [r'^.*\..*$'] + exclusive_regexes = [r'^.*\.adipiscing$'] + expected = set(['ipsum.dolor', 'sit.amet']) + actual = filter_utility.show_tables(table_columns, inclusive_regexes, exclusive_regexes) + self.assertEquals(expected, actual) + + def test_table_names_from_mydumper_backup(self): + """ + Tests `table_names_from_mydumper_backup`. + """ + + files = [] + suffixed_database = 'ipsum.' + expected = [] + actual = filter_utility.table_names_from_mydumper_backup(files, suffixed_database) + self.assertTrue(hasattr(actual, 'next') or hasattr(actual, '__next__')) + self.assertEquals(expected, list(actual)) + + files = ['ipsum.dolor-schema.sql', 'ipsum.dolor.sql', + 'ipsum.consectetur-schema.sql', 'ipsum.consectetur.sql'] + suffixed_database = 'ipsum.' + expected = ['ipsum.dolor.sql', 'ipsum.consectetur.sql'] + actual = filter_utility.table_names_from_mydumper_backup(files, suffixed_database) + self.assertTrue(hasattr(actual, 'next') or hasattr(actual, '__next__')) + self.assertEquals(expected, list(actual)) + + files = ['ipsum.dolor-schema.sql', 'ipsum.dolor.sql', + 'consectetur.adipiscing-schema.sql', 'consectetur.adipiscing.sql'] + suffixed_database = 'ipsum.' + expected = ['ipsum.dolor.sql'] + actual = filter_utility.table_names_from_mydumper_backup(files, suffixed_database) + self.assertEquals(expected, list(actual)) suite = unittest.TestLoader().loadTestsFromTestCase(TestFilter) diff --git a/untraceables/test/test_formatter.py b/untraceables/test/test_formatter.py index 5ad1843..181ec80 100644 --- a/untraceables/test/test_formatter.py +++ b/untraceables/test/test_formatter.py @@ -1,101 +1,101 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import import unittest - from untraceables.utilities import formatter as formatter_utility class TestFormatter(unittest.TestCase): - def test_show_tables(self): - """ - Tests `show_tables`. - """ - - table_columns = () - expected = iter([]) - actual = formatter_utility.show_tables(table_columns) - self.assertTrue(hasattr(actual, 'next')) - self.assertEquals(list(expected), list(actual)) - - table_columns = ({'TABLE_NAME': 'lorem', 'COLUMN_NAME': 'ipsum'},) - expected = iter(['lorem.ipsum']) - actual = formatter_utility.show_tables(table_columns) - self.assertEquals(list(expected), list(actual)) - - table_columns = ({'TABLE_NAME': 'lorem', 'COLUMN_NAME': 'ipsum'}, {'TABLE_NAME': 'dolor', 'COLUMN_NAME': 'sit'}) - expected = iter(['lorem.ipsum', 'dolor.sit']) - actual = formatter_utility.show_tables(table_columns) - self.assertEquals(list(expected), list(actual)) - - def test_table_columns_tsv(self): - """ - Tests `table_columns_tsv`. - """ - - database = 'adipiscing' - table_columns = ['lorem.ipsum', 'dolor.sit'] - expected = ['adipiscing\tlorem\tipsum', 'adipiscing\tdolor\tsit'] - actual = formatter_utility.table_columns_tsv(database, table_columns) - self.assertEquals(list(expected), list(actual)) - - def test_randomize_queries(self): - """ - Tests `randomize_queries`. - """ - - queries = ['SELECT NOW()', 'SELECT 1'] - expected = ('SELECT NOW();\n' - 'SELECT 1;\n') - actual = formatter_utility.randomize_queries(queries) - self.assertEquals(expected, actual) - - queries = [] - expected = '' - actual = formatter_utility.randomize_queries(queries) - self.assertEquals(expected, actual) - - def test_table_names_from_mydumper_backup(self): - """ - Tests `table_names_from_mydumper_backup`. - """ - - files = [] - suffixed_database = 'ipsum.' - actual = formatter_utility.table_names_from_mydumper_backup(files, suffixed_database) - expected = [] - self.assertTrue(hasattr(actual, 'next')) - self.assertEquals(expected, list(actual)) - - files = ['ipsum.dolor.sql', 'ipsum.consectetur.sql'] - suffixed_database = 'ipsum.' - actual = formatter_utility.table_names_from_mydumper_backup(files, suffixed_database) - expected = ['dolor', 'consectetur'] - self.assertTrue(hasattr(actual, 'next')) - self.assertEquals(expected, list(actual)) - - def test_inclusive_regex_in(self): - """ - Tests `inclusive_regex_in`. - """ - - inclusive_regex = r'^ipsum\.id$' - database_table_delimiter = r'\.' - actual = formatter_utility.inclusive_regex_in(inclusive_regex, database_table_delimiter) - expected = r'^ipsum', r'id$' - self.assertEquals(expected, actual) - - def test_inclusive_regex_out(self): - """ - Tests `inclusive_regex_out`. - """ - - file_basename = 'ipsum' - field_regex = r'id$' - database_table_delimiter = r'\.' - actual = formatter_utility.inclusive_regex_out(file_basename, field_regex, database_table_delimiter) - expected = r'^ipsum\.id$' - self.assertEquals(expected, actual) + def test_show_tables(self): + """ + Tests `show_tables`. + """ + + table_columns = () + expected = iter([]) + actual = formatter_utility.show_tables(table_columns) + self.assertTrue(hasattr(actual, 'next') or hasattr(actual, '__next__')) + self.assertEquals(list(expected), list(actual)) + + table_columns = ({'TABLE_NAME': 'lorem', 'COLUMN_NAME': 'ipsum'},) + expected = iter(['lorem.ipsum']) + actual = formatter_utility.show_tables(table_columns) + self.assertEquals(list(expected), list(actual)) + + table_columns = ({'TABLE_NAME': 'lorem', 'COLUMN_NAME': 'ipsum'}, {'TABLE_NAME': 'dolor', 'COLUMN_NAME': 'sit'}) + expected = iter(['lorem.ipsum', 'dolor.sit']) + actual = formatter_utility.show_tables(table_columns) + self.assertEquals(list(expected), list(actual)) + + def test_table_columns_tsv(self): + """ + Tests `table_columns_tsv`. + """ + + database = 'adipiscing' + table_columns = ['lorem.ipsum', 'dolor.sit'] + expected = ['adipiscing\tlorem\tipsum', 'adipiscing\tdolor\tsit'] + actual = formatter_utility.table_columns_tsv(database, table_columns) + self.assertEquals(list(expected), list(actual)) + + def test_randomize_queries(self): + """ + Tests `randomize_queries`. + """ + + queries = ['SELECT NOW()', 'SELECT 1'] + expected = ('SELECT NOW();\n' + 'SELECT 1;\n') + actual = formatter_utility.randomize_queries(queries) + self.assertEquals(expected, actual) + + queries = [] + expected = '' + actual = formatter_utility.randomize_queries(queries) + self.assertEquals(expected, actual) + + def test_table_names_from_mydumper_backup(self): + """ + Tests `table_names_from_mydumper_backup`. + """ + + files = [] + suffixed_database = 'ipsum.' + actual = formatter_utility.table_names_from_mydumper_backup(files, suffixed_database) + expected = [] + self.assertTrue(hasattr(actual, 'next') or hasattr(actual, '__next__')) + self.assertEquals(expected, list(actual)) + + files = ['ipsum.dolor.sql', 'ipsum.consectetur.sql'] + suffixed_database = 'ipsum.' + actual = formatter_utility.table_names_from_mydumper_backup(files, suffixed_database) + expected = ['dolor', 'consectetur'] + self.assertTrue(hasattr(actual, 'next') or hasattr(actual, '__next__')) + self.assertEquals(expected, list(actual)) + + def test_inclusive_regex_in(self): + """ + Tests `inclusive_regex_in`. + """ + + inclusive_regex = r'^ipsum\.id$' + database_table_delimiter = r'\.' + actual = formatter_utility.inclusive_regex_in(inclusive_regex, database_table_delimiter) + expected = r'^ipsum', r'id$' + self.assertEquals(expected, actual) + + def test_inclusive_regex_out(self): + """ + Tests `inclusive_regex_out`. + """ + + file_basename = 'ipsum' + field_regex = r'id$' + database_table_delimiter = r'\.' + actual = formatter_utility.inclusive_regex_out(file_basename, field_regex, database_table_delimiter) + expected = r'^ipsum\.id$' + self.assertEquals(expected, actual) suite = unittest.TestLoader().loadTestsFromTestCase(TestFormatter) diff --git a/untraceables/test/test_mysql.py b/untraceables/test/test_mysql.py index da86fe9..595d771 100644 --- a/untraceables/test/test_mysql.py +++ b/untraceables/test/test_mysql.py @@ -1,11 +1,10 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import import MySQLdb import MySQLdb.cursors - import os import unittest - from untraceables.utilities import mysql as mysql_utility MYSQL_HOST = 'localhost' @@ -31,228 +30,228 @@ class TestMySql(unittest.TestCase): - def test_close_connection_and_cursor_close_not_called(self): - """ - Tests `close_connection_and_cursor`. - - Both connection and cursor have no method close. - """ - - connection = cursor = None - actual = mysql_utility.close_connection_and_cursor(connection, cursor) - self.assertTrue(actual) - - def test_close_connection_and_cursor_close_called_on_connection(self): - """ - Tests `close_connection_and_cursor`. - - Connection has a close method. - """ - - connection = connection_mock() - cursor = None - actual = mysql_utility.close_connection_and_cursor(connection, cursor) - self.assertRaisesRegexp(Warning, 'close called on connection') - - def test_close_connection_and_cursor_close_called_on_cursor(self): - """ - Tests `close_connection_and_cursor`. - - Cursor has a close method. - """ - - connection = None - cursor = cursor_mock() - actual = mysql_utility.close_connection_and_cursor(connection, cursor) - self.assertRaisesRegexp(Warning, 'close called on cursor') - - def test_split_file_0(self): - """ - Tests `split_file`. - - One query per line. - """ - file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'test_split_file_0.sql') - with open(file, 'r') as file_pointer: - delimiter = ';' - actual = mysql_utility.split_file(file_pointer) - self.assertTrue(hasattr(actual, 'next')) - actual_as_list = list(actual) - expected = 'SELECT NOW()' - self.assertEqual(expected, actual_as_list[0]) - expected = '\nSELECT 1' - self.assertEqual(expected, actual_as_list[1]) - expected = '\n' - self.assertEqual(expected, actual_as_list[3]) - - file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'test_split_file_0.sql') - with open(file, 'r') as file_pointer: - delimiter = ';' - actual = mysql_utility.split_file(file_pointer, delimiter) - actual_as_list = list(actual) - expected = 'SELECT NOW()' - self.assertEqual(expected, actual_as_list[0]) - expected = '\nSELECT 1' - self.assertEqual(expected, actual_as_list[1]) - expected = '\n' - self.assertEqual(expected, actual_as_list[3]) - - def test_split_file_1(self): - """ - Tests `split_file`. - - All queries on one line. - """ - - file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'test_split_file_1.sql') - with open(file, 'r') as file_pointer: - delimiter = ';' - actual = mysql_utility.split_file(file_pointer, delimiter) - actual_as_list = list(actual) - expected = 'SELECT NOW()' - self.assertEqual(expected, actual_as_list[0]) - expected = ' SELECT 1' - self.assertEqual(expected, actual_as_list[1]) - expected = '\n' - self.assertEqual(expected, actual_as_list[3]) - - def test_split_file_2(self): - """ - Tests `split_file`. - - - All queries on one line, $ as delimiter. - """ - - file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'test_split_file_2.sql') - with open(file, 'r') as file_pointer: - delimiter = '$' - actual = mysql_utility.split_file(file_pointer, delimiter) - actual_as_list = list(actual) - expected = 'SELECT NOW()' - self.assertEqual(expected, actual_as_list[0]) - expected = ' SELECT 1' - self.assertEqual(expected, actual_as_list[1]) - expected = '\n' - self.assertEqual(expected, actual_as_list[3]) - - def test_get_connection_success(self): - """ - Tests `get_connection`. - - Success. - """ - - actual = None - try: - actual = mysql_utility.get_connection(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DATABASE) - self.assertIsInstance(actual, MySQLdb.connection) - except Exception, e: - raise e - finally: - mysql_utility.close_connection_and_cursor(actual, None) - - def test_get_connection_failure(self): - """ - Tests `get_connection`. - - Failure. - """ - - try: - actual = mysql_utility.get_connection(MYSQL_HOST, MYSQL_USER, 'lorem', MYSQL_DATABASE) - except Exception, e: - self.assertIsInstance(e, MySQLdb.OperationalError) - - def test_get_cursor_success(self): - """ - Tests `get_cursor`. - - Success. - """ - - connection = actual = None - try: - connection = mysql_utility.get_connection(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DATABASE) - actual = mysql_utility.get_cursor(connection) - self.assertIsInstance(actual, MySQLdb.cursors.SSDictCursor) - except Exception, e: - raise e - finally: - mysql_utility.close_connection_and_cursor(connection, actual) - - def test_get_cursor_failure(self): - """ - Tests `get_cursor`. - - Failure. - """ - - try: - actual = mysql_utility.get_cursor(None) - except Exception, e: - self.assertIsInstance(e, AttributeError) - - def test_get_show_columns(self): - """ - Tests `get_show_columns`. - """ - - connection = actual = None - try: - connection = mysql_utility.get_connection(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DATABASE) - cursor = mysql_utility.get_cursor(connection) - table = 'users' - actual = mysql_utility.get_show_columns(cursor, table) - self.assertIsInstance(actual, tuple) - self.assertEqual('id', actual[0]['Field']) - self.assertEqual('mapped_id', actual[1]['Field']) - except Exception, e: - raise e - finally: - mysql_utility.close_connection_and_cursor(connection, cursor) - - def test_get_show_tables(self): - """ - Tests `get_show_tables`. - """ - - connection = mysql_utility.get_connection(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DATABASE) - cursor = mysql_utility.get_cursor(connection) - actual = mysql_utility.get_show_tables(cursor, MYSQL_DATABASE) - self.assertIsInstance(actual, tuple) - self.assertEqual({'TABLE_NAME': 'users', 'COLUMN_NAME': 'id'}, actual[0]) - self.assertEqual({'TABLE_NAME': 'users', 'COLUMN_NAME': 'mapped_id'}, actual[1]) - - def test_get_max_id(self): - """ - Tests `get_max_id`. - """ - - connection = actual = None - try: - connection = mysql_utility.get_connection(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DATABASE) - cursor = mysql_utility.get_cursor(connection) - expected = 2 - actual = mysql_utility.get_max_id(cursor, MYSQL_DATABASE, 'users', 'id') - self.assertEqual(expected, actual) - expected = 10 - actual = mysql_utility.get_max_id(cursor, MYSQL_DATABASE, 'users', 'mapped_id') - self.assertEqual(expected, actual) - except Exception, e: - raise e - finally: - mysql_utility.close_connection_and_cursor(connection, cursor) + def test_close_connection_and_cursor_close_not_called(self): + """ + Tests `close_connection_and_cursor`. + + Both connection and cursor have no method close. + """ + + connection = cursor = None + actual = mysql_utility.close_connection_and_cursor(connection, cursor) + self.assertTrue(actual) + + def test_close_connection_and_cursor_close_called_on_connection(self): + """ + Tests `close_connection_and_cursor`. + + Connection has a close method. + """ + + connection = connection_mock() + cursor = None + actual = mysql_utility.close_connection_and_cursor(connection, cursor) + self.assertRaisesRegexp(Warning, 'close called on connection') + + def test_close_connection_and_cursor_close_called_on_cursor(self): + """ + Tests `close_connection_and_cursor`. + + Cursor has a close method. + """ + + connection = None + cursor = cursor_mock() + actual = mysql_utility.close_connection_and_cursor(connection, cursor) + self.assertRaisesRegexp(Warning, 'close called on cursor') + + def test_split_file_0(self): + """ + Tests `split_file`. + + One query per line. + """ + file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'test_split_file_0.sql') + with open(file, 'r') as file_pointer: + delimiter = ';' + actual = mysql_utility.split_file(file_pointer) + self.assertTrue(hasattr(actual, 'next') or hasattr(actual, '__next__')) + actual_as_list = list(actual) + expected = 'SELECT NOW()' + self.assertEqual(expected, actual_as_list[0]) + expected = '\nSELECT 1' + self.assertEqual(expected, actual_as_list[1]) + expected = '\n' + self.assertEqual(expected, actual_as_list[3]) + + file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'test_split_file_0.sql') + with open(file, 'r') as file_pointer: + delimiter = ';' + actual = mysql_utility.split_file(file_pointer, delimiter) + actual_as_list = list(actual) + expected = 'SELECT NOW()' + self.assertEqual(expected, actual_as_list[0]) + expected = '\nSELECT 1' + self.assertEqual(expected, actual_as_list[1]) + expected = '\n' + self.assertEqual(expected, actual_as_list[3]) + + def test_split_file_1(self): + """ + Tests `split_file`. + + All queries on one line. + """ + + file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'test_split_file_1.sql') + with open(file, 'r') as file_pointer: + delimiter = ';' + actual = mysql_utility.split_file(file_pointer, delimiter) + actual_as_list = list(actual) + expected = 'SELECT NOW()' + self.assertEqual(expected, actual_as_list[0]) + expected = ' SELECT 1' + self.assertEqual(expected, actual_as_list[1]) + expected = '\n' + self.assertEqual(expected, actual_as_list[3]) + + def test_split_file_2(self): + """ + Tests `split_file`. + + + All queries on one line, $ as delimiter. + """ + + file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'test_split_file_2.sql') + with open(file, 'r') as file_pointer: + delimiter = '$' + actual = mysql_utility.split_file(file_pointer, delimiter) + actual_as_list = list(actual) + expected = 'SELECT NOW()' + self.assertEqual(expected, actual_as_list[0]) + expected = ' SELECT 1' + self.assertEqual(expected, actual_as_list[1]) + expected = '\n' + self.assertEqual(expected, actual_as_list[3]) + + def test_get_connection_success(self): + """ + Tests `get_connection`. + + Success. + """ + + actual = None + try: + actual = mysql_utility.get_connection(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DATABASE) + self.assertIsInstance(actual, MySQLdb.connection) + except Exception as e: + raise e + finally: + mysql_utility.close_connection_and_cursor(actual, None) + + def test_get_connection_failure(self): + """ + Tests `get_connection`. + + Failure. + """ + + try: + actual = mysql_utility.get_connection(MYSQL_HOST, MYSQL_USER, 'lorem', MYSQL_DATABASE) + except Exception as e: + self.assertIsInstance(e, MySQLdb.OperationalError) + + def test_get_cursor_success(self): + """ + Tests `get_cursor`. + + Success. + """ + + connection = actual = None + try: + connection = mysql_utility.get_connection(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DATABASE) + actual = mysql_utility.get_cursor(connection) + self.assertIsInstance(actual, MySQLdb.cursors.SSDictCursor) + except Exception as e: + raise e + finally: + mysql_utility.close_connection_and_cursor(connection, actual) + + def test_get_cursor_failure(self): + """ + Tests `get_cursor`. + + Failure. + """ + + try: + actual = mysql_utility.get_cursor(None) + except Exception as e: + self.assertIsInstance(e, AttributeError) + + def test_get_show_columns(self): + """ + Tests `get_show_columns`. + """ + + connection = actual = None + try: + connection = mysql_utility.get_connection(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DATABASE) + cursor = mysql_utility.get_cursor(connection) + table = 'users' + actual = mysql_utility.get_show_columns(cursor, table) + self.assertIsInstance(actual, tuple) + self.assertEqual('id', actual[0]['Field']) + self.assertEqual('mapped_id', actual[1]['Field']) + except Exception as e: + raise e + finally: + mysql_utility.close_connection_and_cursor(connection, cursor) + + def test_get_show_tables(self): + """ + Tests `get_show_tables`. + """ + + connection = mysql_utility.get_connection(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DATABASE) + cursor = mysql_utility.get_cursor(connection) + actual = mysql_utility.get_show_tables(cursor, MYSQL_DATABASE) + self.assertIsInstance(actual, tuple) + self.assertEqual({'TABLE_NAME': 'users', 'COLUMN_NAME': 'id'}, actual[0]) + self.assertEqual({'TABLE_NAME': 'users', 'COLUMN_NAME': 'mapped_id'}, actual[1]) + + def test_get_max_id(self): + """ + Tests `get_max_id`. + """ + + connection = actual = None + try: + connection = mysql_utility.get_connection(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DATABASE) + cursor = mysql_utility.get_cursor(connection) + expected = 2 + actual = mysql_utility.get_max_id(cursor, MYSQL_DATABASE, 'users', 'id') + self.assertEqual(expected, actual) + expected = 10 + actual = mysql_utility.get_max_id(cursor, MYSQL_DATABASE, 'users', 'mapped_id') + self.assertEqual(expected, actual) + except Exception as e: + raise e + finally: + mysql_utility.close_connection_and_cursor(connection, cursor) class connection_mock(object): - def close(self): - raise Warning('close called on connection') + def close(self): + raise Warning('close called on connection') class cursor_mock(object): - def close(self): - raise Warning('close called on cursor') + def close(self): + raise Warning('close called on cursor') suite = unittest.TestLoader().loadTestsFromTestCase(TestMySql) diff --git a/untraceables/test/test_query.py b/untraceables/test/test_query.py index 22b6aa4..f3eb400 100644 --- a/untraceables/test/test_query.py +++ b/untraceables/test/test_query.py @@ -1,115 +1,115 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import import unittest - from untraceables.utilities import query as query_utility class TestQuery(unittest.TestCase): - def test_get_show_table_columns(self): - """ - Tests `get_show_table_columns`. - """ - - table = 'lorem' - actual = query_utility.get_show_table_columns(table) - expected = 'SHOW COLUMNS FROM `lorem`' - self.assertEqual(expected, actual) - - def test_get_show_columns(self): - """ - Tests `get_show_columns`. - """ - - database = 'lorem' - actual = query_utility.get_show_columns(database) - expected = 'SELECT `TABLE_NAME`, `COLUMN_NAME`' - self.assertTrue(expected in actual) - expected = 'WHERE `TABLE_SCHEMA` = \'lorem\'' - self.assertTrue(expected in actual) - - def test_get_max_id(self): - """ - Tests `get_max_id`. - """ - - database = 'lorem' - table = 'ipsum' - column = 'dolor' - order = 'sit' - - actual = query_utility.get_max_id(database, table, column) - expected = 'SELECT `dolor` FROM `lorem`.`ipsum` ORDER BY `dolor` DESC LIMIT 1' - self.assertEqual(expected, actual) - actual = query_utility.get_max_id(database, table, column, order) - expected = 'SELECT `dolor` FROM `lorem`.`ipsum` ORDER BY `sit` DESC LIMIT 1' - self.assertEqual(expected, actual) - - def test_get_foreign_key_checks(self): - """ - Tests `get_foreign_key_checks`. - """ - - actual = query_utility.get_foreign_key_checks(True) - expected = 'SET FOREIGN_KEY_CHECKS=1' - self.assertEqual(expected, actual) - actual = query_utility.get_foreign_key_checks(1) - self.assertEqual(expected, actual) - - actual = query_utility.get_foreign_key_checks(False) - expected = 'SET FOREIGN_KEY_CHECKS=0' - self.assertEqual(expected, actual) - actual = query_utility.get_foreign_key_checks(0) - self.assertEqual(expected, actual) - - def test_get_unique_checks(self): - """ - Tests `get_unique_checks`. - """ - - actual = query_utility.get_unique_checks(True) - expected = 'SET UNIQUE_CHECKS=1' - self.assertEqual(expected, actual) - actual = query_utility.get_unique_checks(1) - self.assertEqual(expected, actual) - - actual = query_utility.get_unique_checks(False) - expected = 'SET UNIQUE_CHECKS=0' - self.assertEqual(expected, actual) - actual = query_utility.get_unique_checks(0) - self.assertEqual(expected, actual) - - def test_get_randomize(self): - """ - Tests `get_randomize`. - """ - - database = 'lorem' - table = 'ipsum' - columns = ({'Field': 'dolor'}, {'Field': 'sit'}, {'Field': 'amet'}) - column = 'consectetur' - mapping_database = 'adipiscing' - mapping_table = 'elit' - - expected = type(list()) - actual = query_utility.get_randomize(database, table, columns, column, mapping_database, mapping_table) - self.assertEqual(expected, type(actual)) - self.assertTrue(len(actual) > 0) - - expected = 'DROP TABLE IF EXISTS `lorem`.`_ipsum`' - self.assertEqual(expected, actual[0]) - expected = 'CREATE TABLE `lorem`.`_ipsum` LIKE `lorem`.`ipsum`' - self.assertEqual(expected, actual[1]) - expected = ('INSERT INTO `lorem`.`_ipsum` ' - 'SELECT `t1`.`dolor`, `t1`.`sit`, `t1`.`amet` ' - 'FROM `lorem`.`ipsum` `t1` ' - 'LEFT JOIN `adipiscing`.`elit` `t2` ON `t2`.`id` = `t1`.`consectetur`') - self.assertEqual(expected, actual[2]) - expected = 'DROP TABLE `lorem`.`ipsum`' - self.assertEqual(expected, actual[3]) - expected = 'RENAME TABLE `lorem`.`_ipsum` TO `lorem`.`ipsum`' - self.assertEqual(expected, actual[4]) + def test_get_show_table_columns(self): + """ + Tests `get_show_table_columns`. + """ + + table = 'lorem' + actual = query_utility.get_show_table_columns(table) + expected = 'SHOW COLUMNS FROM `lorem`' + self.assertEqual(expected, actual) + + def test_get_show_columns(self): + """ + Tests `get_show_columns`. + """ + + database = 'lorem' + actual = query_utility.get_show_columns(database) + expected = 'SELECT `TABLE_NAME`, `COLUMN_NAME`' + self.assertTrue(expected in actual) + expected = 'WHERE `TABLE_SCHEMA` = \'lorem\'' + self.assertTrue(expected in actual) + + def test_get_max_id(self): + """ + Tests `get_max_id`. + """ + + database = 'lorem' + table = 'ipsum' + column = 'dolor' + order = 'sit' + + actual = query_utility.get_max_id(database, table, column) + expected = 'SELECT `dolor` FROM `lorem`.`ipsum` ORDER BY `dolor` DESC LIMIT 1' + self.assertEqual(expected, actual) + actual = query_utility.get_max_id(database, table, column, order) + expected = 'SELECT `dolor` FROM `lorem`.`ipsum` ORDER BY `sit` DESC LIMIT 1' + self.assertEqual(expected, actual) + + def test_get_foreign_key_checks(self): + """ + Tests `get_foreign_key_checks`. + """ + + actual = query_utility.get_foreign_key_checks(True) + expected = 'SET FOREIGN_KEY_CHECKS=1' + self.assertEqual(expected, actual) + actual = query_utility.get_foreign_key_checks(1) + self.assertEqual(expected, actual) + + actual = query_utility.get_foreign_key_checks(False) + expected = 'SET FOREIGN_KEY_CHECKS=0' + self.assertEqual(expected, actual) + actual = query_utility.get_foreign_key_checks(0) + self.assertEqual(expected, actual) + + def test_get_unique_checks(self): + """ + Tests `get_unique_checks`. + """ + + actual = query_utility.get_unique_checks(True) + expected = 'SET UNIQUE_CHECKS=1' + self.assertEqual(expected, actual) + actual = query_utility.get_unique_checks(1) + self.assertEqual(expected, actual) + + actual = query_utility.get_unique_checks(False) + expected = 'SET UNIQUE_CHECKS=0' + self.assertEqual(expected, actual) + actual = query_utility.get_unique_checks(0) + self.assertEqual(expected, actual) + + def test_get_randomize(self): + """ + Tests `get_randomize`. + """ + + database = 'lorem' + table = 'ipsum' + columns = ({'Field': 'dolor'}, {'Field': 'sit'}, {'Field': 'amet'}) + column = 'consectetur' + mapping_database = 'adipiscing' + mapping_table = 'elit' + + expected = type(list()) + actual = query_utility.get_randomize(database, table, columns, column, mapping_database, mapping_table) + self.assertEqual(expected, type(actual)) + self.assertTrue(len(actual) > 0) + + expected = 'DROP TABLE IF EXISTS `lorem`.`_ipsum`' + self.assertEqual(expected, actual[0]) + expected = 'CREATE TABLE `lorem`.`_ipsum` LIKE `lorem`.`ipsum`' + self.assertEqual(expected, actual[1]) + expected = ('INSERT INTO `lorem`.`_ipsum` ' + 'SELECT `t1`.`dolor`, `t1`.`sit`, `t1`.`amet` ' + 'FROM `lorem`.`ipsum` `t1` ' + 'LEFT JOIN `adipiscing`.`elit` `t2` ON `t2`.`id` = `t1`.`consectetur`') + self.assertEqual(expected, actual[2]) + expected = 'DROP TABLE `lorem`.`ipsum`' + self.assertEqual(expected, actual[3]) + expected = 'RENAME TABLE `lorem`.`_ipsum` TO `lorem`.`ipsum`' + self.assertEqual(expected, actual[4]) suite = unittest.TestLoader().loadTestsFromTestCase(TestQuery) diff --git a/untraceables/test/test_validation.py b/untraceables/test/test_validation.py index 70efab5..e0341f6 100644 --- a/untraceables/test/test_validation.py +++ b/untraceables/test/test_validation.py @@ -1,40 +1,40 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import import unittest - from untraceables.utilities import validation as validation_utility class TestValidation(unittest.TestCase): - def test_check_max_ids(self): - """ - Tests `check_max_ids`. - """ - - max_id = mapping_max_id = None - actual = validation_utility.check_max_ids(max_id, mapping_max_id) - self.assertTrue(actual) - - max_id = 10 - mapping_max_id = None - actual = validation_utility.check_max_ids(max_id, mapping_max_id) - self.assertTrue(actual) - - max_id = 10 - mapping_max_id = 9 - actual = validation_utility.check_max_ids(max_id, mapping_max_id) - self.assertTrue(actual) - - max_id = 10 - mapping_max_id = 10 - actual = validation_utility.check_max_ids(max_id, mapping_max_id) - self.assertFalse(actual) - - max_id = None - mapping_max_id = 10 - actual = validation_utility.check_max_ids(max_id, mapping_max_id) - self.assertFalse(actual) + def test_check_max_ids(self): + """ + Tests `check_max_ids`. + """ + + max_id = mapping_max_id = None + actual = validation_utility.check_max_ids(max_id, mapping_max_id) + self.assertTrue(actual) + + max_id = 10 + mapping_max_id = None + actual = validation_utility.check_max_ids(max_id, mapping_max_id) + self.assertTrue(actual) + + max_id = 10 + mapping_max_id = 9 + actual = validation_utility.check_max_ids(max_id, mapping_max_id) + self.assertTrue(actual) + + max_id = 10 + mapping_max_id = 10 + actual = validation_utility.check_max_ids(max_id, mapping_max_id) + self.assertFalse(actual) + + max_id = None + mapping_max_id = 10 + actual = validation_utility.check_max_ids(max_id, mapping_max_id) + self.assertFalse(actual) suite = unittest.TestLoader().loadTestsFromTestCase(TestValidation) diff --git a/untraceables/utilities/cli.py b/untraceables/utilities/cli.py index 48ad8e7..d206820 100644 --- a/untraceables/utilities/cli.py +++ b/untraceables/utilities/cli.py @@ -5,10 +5,9 @@ """ from __future__ import print_function - +from __future__ import absolute_import import sys - CONFIGURATION_FILE = 'untraceables.cfg' """ The name of untraceables' configuration file. @@ -18,25 +17,25 @@ def config_unpack(config): - """ - Unpacks relevant options from the configuration object. + """ + Unpacks relevant options from the configuration object. - :type config: configobj.ConfigObj - :param config: A configuration object - :rtype tuple - :return Relevant options - """ + :type config: configobj.ConfigObj + :param config: A configuration object + :rtype tuple + :return Relevant options + """ - return config['main']['host'], config['main']['user'], config['main']['password'] + return config['main']['host'], config['main']['user'], config['main']['password'] def print_e(error): - """ - Prints an error to `STDERR` and exits with a return code of `1`. + """ + Prints an error to `STDERR` and exits with a return code of `1`. - :type error: mixed - :param error: An error - """ + :type error: mixed + :param error: An error + """ - print(error, file=sys.stderr) - sys.exit(1) + print(error, file=sys.stderr) + sys.exit(1) diff --git a/untraceables/utilities/configuration.py b/untraceables/utilities/configuration.py index 55142e3..5a9828c 100644 --- a/untraceables/utilities/configuration.py +++ b/untraceables/utilities/configuration.py @@ -4,50 +4,50 @@ Configuration utility functions. """ +from __future__ import absolute_import import os - from configobj import ConfigObj def read_file(filename): - """ - Reads a configuration file and returns the configuration object (on success). + """ + Reads a configuration file and returns the configuration object (on success). - Looks for the given configuration file in: - * The current directory (.) - * The users home directory (~) - * Globally (/etc) + Looks for the given configuration file in: + * The current directory (.) + * The users home directory (~) + * Globally (/etc) - :type filename: str - :param filename: A (configuration) file name - :rtype bool|configobj.ConfigObj - :return A configuration object or False on failure - """ + :type filename: str + :param filename: A (configuration) file name + :rtype bool|configobj.ConfigObj + :return A configuration object or False on failure + """ - for path in os.curdir, os.path.expanduser('~'), '/etc/': - try: - with open(os.path.join(path, filename)) as filepointer: - return ConfigObj(filepointer) - except IOError: - pass + for path in os.curdir, os.path.expanduser('~'), '/etc/': + try: + with open(os.path.join(path, filename)) as filepointer: + return ConfigObj(filepointer) + except IOError: + pass - return False + return False def read_xclude_regexes_file(filename): - """ - Reads an ((in|ex)clude regexes) file and returns it's lines. - - :type filename: str - :param filename: An ((in|ex)clude regexes) file name - :rtype list - :return File content lines - """ - - try: - with open(filename) as filepointer: - return filepointer.read().splitlines() - except IOError: - pass - - return [] + """ + Reads an ((in|ex)clude regexes) file and returns it's lines. + + :type filename: str + :param filename: An ((in|ex)clude regexes) file name + :rtype list + :return File content lines + """ + + try: + with open(filename) as filepointer: + return filepointer.read().splitlines() + except IOError: + pass + + return [] diff --git a/untraceables/utilities/file.py b/untraceables/utilities/file.py index 919e081..321dd97 100644 --- a/untraceables/utilities/file.py +++ b/untraceables/utilities/file.py @@ -4,21 +4,22 @@ File utility functions. """ +from __future__ import absolute_import import os def get_sorted_file_list(path): - """ - Gets a sorted directory listing (files only) for a given path. + """ + Gets a sorted directory listing (files only) for a given path. - :type path: string - :param path: A path - :rtype list - :return Sorted file names - """ + :type path: string + :param path: A path + :rtype list + :return Sorted file names + """ - if os.path.exists(path): - files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))] - return sorted(files) + if os.path.exists(path): + files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))] + return sorted(files) - return [] + return [] diff --git a/untraceables/utilities/filter.py b/untraceables/utilities/filter.py index 51a0bc6..2985ed8 100644 --- a/untraceables/utilities/filter.py +++ b/untraceables/utilities/filter.py @@ -4,66 +4,67 @@ Filter utility functions. """ +from __future__ import absolute_import import re def _get_matches(input_columns, regexes): - """ - Returns all of input columns where `table.column` matches a given set of regexes. + """ + Returns all of input columns where `table.column` matches a given set of regexes. - :type input_columns: generator|set - :param input_columns: Input columns - :type regexes: list - :param regexes: Zero or more regexes - :rtype set - :return Matches - """ + :type input_columns: generator|set + :param input_columns: Input columns + :type regexes: list + :param regexes: Zero or more regexes + :rtype set + :return Matches + """ - patterns = [re.compile(r) for r in regexes] + patterns = [re.compile(r) for r in regexes] - matched_columns = set() - for input_column in input_columns: - for pattern in patterns: - if re.match(pattern, input_column): - matched_columns.add(input_column) + matched_columns = set() + for input_column in input_columns: + for pattern in patterns: + if re.match(pattern, input_column): + matched_columns.add(input_column) - return matched_columns + return matched_columns def show_tables(table_columns, inclusive_regexes, exclusive_regexes): - """ - Returns all columns that match a given set of inclusive regexes and don't match the exclusive regexes. + """ + Returns all columns that match a given set of inclusive regexes and don't match the exclusive regexes. - Matching is done using `_get_matches`. + Matching is done using `_get_matches`. - :type table_columns: generator - :param table_columns: Input columns (in format `table.column`) - :type inclusive_regexes: list - :param inclusive_regexes: Zero or more inclusive regexes - :type exclusive_regexes: list - :param exclusive_regexes: Zero or more exclusive regexes - :rtype set - :return Matches - """ + :type table_columns: generator + :param table_columns: Input columns (in format `table.column`) + :type inclusive_regexes: list + :param inclusive_regexes: Zero or more inclusive regexes + :type exclusive_regexes: list + :param exclusive_regexes: Zero or more exclusive regexes + :rtype set + :return Matches + """ - included_columns = _get_matches(table_columns, inclusive_regexes) - excluded_columns = _get_matches(included_columns, exclusive_regexes) + included_columns = _get_matches(table_columns, inclusive_regexes) + excluded_columns = _get_matches(included_columns, exclusive_regexes) - return included_columns - excluded_columns + return included_columns - excluded_columns def table_names_from_mydumper_backup(files, suffixed_database): - """ - Filters table names from a directory listing (mydumper backup). - - :type files: list - :param files: File names - :type suffixed_database: string - :param suffixed_database: A database name suffixes with a `.` (e.g. `example_com_www.`) - :rtype generator - :return Table names - """ - - for file_name in files: - if file_name.startswith(suffixed_database) and not file_name.endswith('schema.sql'): - yield file_name + """ + Filters table names from a directory listing (mydumper backup). + + :type files: list + :param files: File names + :type suffixed_database: string + :param suffixed_database: A database name suffixes with a `.` (e.g. `example_com_www.`) + :rtype generator + :return Table names + """ + + for file_name in files: + if file_name.startswith(suffixed_database) and not file_name.endswith('schema.sql'): + yield file_name diff --git a/untraceables/utilities/formatter.py b/untraceables/utilities/formatter.py index 2fc5e16..4b5ccd8 100644 --- a/untraceables/utilities/formatter.py +++ b/untraceables/utilities/formatter.py @@ -4,109 +4,110 @@ Formatter utility functions. """ +from __future__ import absolute_import import os def show_tables(table_columns): - """ - Formats the results of SHOW TABLES. + """ + Formats the results of SHOW TABLES. - In the format `table.column`. + In the format `table.column`. - :type tuple - :param table_columns: Unformatted table columns - :rtype generator - :return Formatted table columns - """ + :type tuple + :param table_columns: Unformatted table columns + :rtype generator + :return Formatted table columns + """ - for table_column in table_columns: - yield '{0:s}.{1:s}'.format(table_column['TABLE_NAME'], table_column['COLUMN_NAME']) + for table_column in table_columns: + yield '{0:s}.{1:s}'.format(table_column['TABLE_NAME'], table_column['COLUMN_NAME']) def table_columns_tsv(database, table_columns): - """ - Formats the results of SHOW COLUMNS. + """ + Formats the results of SHOW COLUMNS. - In the format `databasetablecolumn`. + In the format `databasetablecolumn`. - :type str - :param database: A database name - :type list - :param table_columns: Unformatted table columns - :rtype generator - :return Formatted table columns - """ + :type str + :param database: A database name + :type list + :param table_columns: Unformatted table columns + :rtype generator + :return Formatted table columns + """ - for table_column in table_columns: - table, column = table_column.split('.') - yield '\t'.join((database, table, column)) + for table_column in table_columns: + table, column = table_column.split('.') + yield '\t'.join((database, table, column)) def randomize_queries(queries): - """ - Formats the results `query.get_randomize`. + """ + Formats the results `query.get_randomize`. - :type list - :param queries: Unformatted queries - :rtype str - :return Formatted queries - """ + :type list + :param queries: Unformatted queries + :rtype str + :return Formatted queries + """ - if len(queries) > 0: - separator = ';' + os.linesep + if len(queries) > 0: + separator = ';' + os.linesep - return separator.join(queries) + separator + return separator.join(queries) + separator - return '' + return '' def table_names_from_mydumper_backup(files, suffixed_database): - """ - Formats the results `filter.table_names_from_mydumper_backup`. + """ + Formats the results `filter.table_names_from_mydumper_backup`. - :type files: generator - :param files: Filtered file names - :type suffixed_database: string - :param suffixed_database: A database name suffixes with a `.` (e.g. `example_com_www.`) - :rtype generator - :return Table names - """ + :type files: generator + :param files: Filtered file names + :type suffixed_database: string + :param suffixed_database: A database name suffixes with a `.` (e.g. `example_com_www.`) + :rtype generator + :return Table names + """ - for file_name in files: - yield os.path.splitext(file_name)[0].replace(suffixed_database, '') + for file_name in files: + yield os.path.splitext(file_name)[0].replace(suffixed_database, '') def inclusive_regex_in(inclusive_regex, database_table_delimiter): - r""" - Formats an inclusive regex (for input). + r""" + Formats an inclusive regex (for input). - :type inclusive_regex: str - :param inclusive_regex: An inclusive regex (e.g. `^users\.id$`) - :type database_table_delimiter: str - :param database_table_delimiter: A database table delimiter for use in regex (e.g. `\.`) - :rtype tuple - :return Splitted table and field regex - """ + :type inclusive_regex: str + :param inclusive_regex: An inclusive regex (e.g. `^users\.id$`) + :type database_table_delimiter: str + :param database_table_delimiter: A database table delimiter for use in regex (e.g. `\.`) + :rtype tuple + :return Splitted table and field regex + """ - splitted_regex = inclusive_regex.split(database_table_delimiter) - table_regex = splitted_regex[0] - field_regex = database_table_delimiter.join(splitted_regex[1:]) + splitted_regex = inclusive_regex.split(database_table_delimiter) + table_regex = splitted_regex[0] + field_regex = database_table_delimiter.join(splitted_regex[1:]) - return table_regex, field_regex + return table_regex, field_regex def inclusive_regex_out(file_basename, field_regex, database_table_delimiter): - r""" - Formats an inclusive regex (for output). - - :type file_basename: str - :param file_basename: A file (base)name (e.g. `users`) - :type field_regex: str - :param field_regex: A field regex (e.g. `id$`) - :type database_table_delimiter: str - :param database_table_delimiter: A database table delimiter for use in regex (e.g. `\.`) - :rtype str - :return An inclusive regex (e.g. `^users\.id$`) - """ - - return database_table_delimiter.join(['^' + file_basename, field_regex]) + r""" + Formats an inclusive regex (for output). + + :type file_basename: str + :param file_basename: A file (base)name (e.g. `users`) + :type field_regex: str + :param field_regex: A field regex (e.g. `id$`) + :type database_table_delimiter: str + :param database_table_delimiter: A database table delimiter for use in regex (e.g. `\.`) + :rtype str + :return An inclusive regex (e.g. `^users\.id$`) + """ + + return database_table_delimiter.join(['^' + file_basename, field_regex]) diff --git a/untraceables/utilities/mysql.py b/untraceables/utilities/mysql.py index a743465..4f44838 100644 --- a/untraceables/utilities/mysql.py +++ b/untraceables/utilities/mysql.py @@ -4,159 +4,158 @@ MySQL utility functions. """ +from __future__ import absolute_import from warnings import filterwarnings - import MySQLdb import MySQLdb.cursors - from untraceables.utilities import query as query_utility filterwarnings('ignore', category=MySQLdb.Warning) def get_connection(host, user, password, database): - """ - Gets a mysql connection to given database. + """ + Gets a mysql connection to given database. - :type str - :param host: A host - :type str - :param user: A username - :type str - :param password: A password - :type str - :param database: A database name - :rtype MySQLdb.connections.Connection - :return A mysql connection - """ + :type str + :param host: A host + :type str + :param user: A username + :type str + :param password: A password + :type str + :param database: A database name + :rtype MySQLdb.connections.Connection + :return A mysql connection + """ - return MySQLdb.connect(host=host, user=user, passwd=password, db=database, cursorclass=MySQLdb.cursors.SSDictCursor) + return MySQLdb.connect(host=host, user=user, passwd=password, db=database, cursorclass=MySQLdb.cursors.SSDictCursor) def get_cursor(connection): - """ - Gets a cursor from a given connection. + """ + Gets a cursor from a given connection. - :type MySQLdb.connections.Connection - :param connection: A mysql connection - :rtype MySQLdb.cursors.SSDictCursor - :return A mysql cursor - """ + :type MySQLdb.connections.Connection + :param connection: A mysql connection + :rtype MySQLdb.cursors.SSDictCursor + :return A mysql cursor + """ - return connection.cursor() + return connection.cursor() def close_connection_and_cursor(connection, cursor): - """ - Closes a given connection and cursor. + """ + Closes a given connection and cursor. - :type MySQLdb.connections.Connection - :param connection: A mysql connection - :type MySQLdb.cursors.SSDictCursor - :param cursor: A mysql cursor - :rtype bool - :return: Success - """ + :type MySQLdb.connections.Connection + :param connection: A mysql connection + :type MySQLdb.cursors.SSDictCursor + :param cursor: A mysql cursor + :rtype bool + :return: Success + """ - attr = 'close' - for to_be_closed_o in (cursor, connection): - if hasattr(to_be_closed_o, attr): - getattr(to_be_closed_o, attr) + attr = 'close' + for to_be_closed_o in (cursor, connection): + if hasattr(to_be_closed_o, attr): + getattr(to_be_closed_o, attr) - return True + return True def split_file(file_pointer, delimiter=';'): - """ - Splits a SQL file by a given delimiter so it can be executed statement by statement. - - :type file - :param file_pointer: The file pointer to an unsplitted SQL file - :type str - :param delimiter: A delimiter - :rtype generator - :return A splitted SQL file - """ - - buf = '' - while True: - while delimiter in buf: - pos = buf.index(delimiter) - yield buf[:pos] - buf = buf[pos + len(delimiter):] - chunk = file_pointer.read(4096) - if not chunk: - yield buf - break - buf += chunk + """ + Splits a SQL file by a given delimiter so it can be executed statement by statement. + + :type file + :param file_pointer: The file pointer to an unsplitted SQL file + :type str + :param delimiter: A delimiter + :rtype generator + :return A splitted SQL file + """ + + buf = '' + while True: + while delimiter in buf: + pos = buf.index(delimiter) + yield buf[:pos] + buf = buf[pos + len(delimiter):] + chunk = file_pointer.read(4096) + if not chunk: + yield buf + break + buf += chunk def get_show_columns(cursor, table): - """ - Gets the results of SHOW COLUMNS for a given table. + """ + Gets the results of SHOW COLUMNS for a given table. - :type MySQLdb.cursors.SSDictCursor - :param cursor: A mysql cursor - :type str - :param table: A table name - :rtype tuple - :return The results of SHOW COLUMNS - """ + :type MySQLdb.cursors.SSDictCursor + :param cursor: A mysql cursor + :type str + :param table: A table name + :rtype tuple + :return The results of SHOW COLUMNS + """ - return _fetchall(cursor, query_utility.get_show_table_columns(table)) + return _fetchall(cursor, query_utility.get_show_table_columns(table)) def get_show_tables(cursor, database): - """ - Gets the results of SHOW TABLES for a given database. + """ + Gets the results of SHOW TABLES for a given database. - :type MySQLdb.cursors.SSDictCursor - :param cursor: A mysql cursor - :type str - :param database: A database name - :rtype tuple - :return The results of SHOW TABLES - """ + :type MySQLdb.cursors.SSDictCursor + :param cursor: A mysql cursor + :type str + :param database: A database name + :rtype tuple + :return The results of SHOW TABLES + """ - return _fetchall(cursor, query_utility.get_show_columns(database)) + return _fetchall(cursor, query_utility.get_show_columns(database)) def get_max_id(cursor, database, table, column, order=None): - """ - Gets the maximum id for a given table / column. + """ + Gets the maximum id for a given table / column. - :type MySQLdb.cursors.SSDictCursor - :param cursor: A mysql cursor - :type str - :param database: A database name - :type str - :param table: A table name - :type str - :param column: A column name - :type str - :param order: A column name to order on - :rtype long|bool - :return: The maximum id or False on failure - """ + :type MySQLdb.cursors.SSDictCursor + :param cursor: A mysql cursor + :type str + :param database: A database name + :type str + :param table: A table name + :type str + :param column: A column name + :type str + :param order: A column name to order on + :rtype long|bool + :return: The maximum id or False on failure + """ - for row in _fetchall(cursor, query_utility.get_max_id(database, table, column, order)): - return row[column] + for row in _fetchall(cursor, query_utility.get_max_id(database, table, column, order)): + return row[column] - return False + return False def _fetchall(cursor, statement): - """ - Fetches the results of a given SQL statement. + """ + Fetches the results of a given SQL statement. - :type MySQLdb.cursors.SSDictCursor - :param cursor: A mysql cursor - :type str - :param statement: A SQL statement - :rtype tuple - :return The results of the MySQLdb statement - """ + :type MySQLdb.cursors.SSDictCursor + :param cursor: A mysql cursor + :type str + :param statement: A SQL statement + :rtype tuple + :return The results of the MySQLdb statement + """ - cursor.execute(statement) + cursor.execute(statement) - return cursor.fetchall() + return cursor.fetchall() diff --git a/untraceables/utilities/query.py b/untraceables/utilities/query.py index 0ec4e24..cc3ff6c 100644 --- a/untraceables/utilities/query.py +++ b/untraceables/utilities/query.py @@ -4,154 +4,155 @@ Query utility functions. """ +from __future__ import absolute_import import untraceables def get_show_table_columns(table): - """ - Gets the query of SHOW COLUMNS for a given table. + """ + Gets the query of SHOW COLUMNS for a given table. - :type str - :param table: A table name - :rtype str - :return A query - """ + :type str + :param table: A table name + :rtype str + :return A query + """ - return 'SHOW COLUMNS FROM `{:s}`'.format(table) + return 'SHOW COLUMNS FROM `{:s}`'.format(table) def get_show_columns(database): - """ - Gets the query of SHOW COLUMNS for a given database. + """ + Gets the query of SHOW COLUMNS for a given database. - :type str - :param database: A database name - :rtype str - :return A query - """ + :type str + :param database: A database name + :rtype str + :return A query + """ - return ("SELECT `TABLE_NAME`, `COLUMN_NAME` " - " FROM " - "`information_schema`.`COLUMNS`" - " WHERE " - "`TABLE_SCHEMA` = '{:s}'").format(database) + return ("SELECT `TABLE_NAME`, `COLUMN_NAME` " + " FROM " + "`information_schema`.`COLUMNS`" + " WHERE " + "`TABLE_SCHEMA` = '{:s}'").format(database) def get_max_id(database, table, column, order=None): - """ - Gets the query to determine the maximum id for a given table / column. + """ + Gets the query to determine the maximum id for a given table / column. - :type str - :param database: A database name - :type str - :param table: A table name - :type str - :param column: A column name - :type str - :param order: A column name to order on - :rtype long - :return The maximum id - """ + :type str + :param database: A database name + :type str + :param table: A table name + :type str + :param column: A column name + :type str + :param order: A column name to order on + :rtype long + :return The maximum id + """ - if not order: - order = column + if not order: + order = column - return 'SELECT `{:s}` FROM `{:s}`.`{:s}` ORDER BY `{:s}` DESC LIMIT 1'.format(column, database, table, order) + return 'SELECT `{:s}` FROM `{:s}`.`{:s}` ORDER BY `{:s}` DESC LIMIT 1'.format(column, database, table, order) def get_foreign_key_checks(enabled): - """ - Gets the query the enable / disable FOREIGN_KEY_CHECKS. + """ + Gets the query the enable / disable FOREIGN_KEY_CHECKS. - :type bool - :param enabled: Whether or not to enable - :rtype str - :return A query - """ + :type bool + :param enabled: Whether or not to enable + :rtype str + :return A query + """ - return 'SET FOREIGN_KEY_CHECKS={0:d}'.format(enabled) + return 'SET FOREIGN_KEY_CHECKS={0:d}'.format(enabled) def get_unique_checks(enabled): - """ - Gets the query the enable / disable UNIQUE_CHECKS. + """ + Gets the query the enable / disable UNIQUE_CHECKS. - :type bool - :param enabled: Whether or not to enable - :rtype str - :return A query - """ + :type bool + :param enabled: Whether or not to enable + :rtype str + :return A query + """ - return 'SET UNIQUE_CHECKS={0:d}'.format(enabled) + return 'SET UNIQUE_CHECKS={0:d}'.format(enabled) def get_randomize(database, table, columns, column, mapping_database, mapping_table): - """ - Gets the queries to randomize a table / column in a given database. - - :type str - :param database: A database name - :type str - :param table: A table name - :type tuple - :param columns: Zero or more column names (result of `SHOW COLUMNS`) - :type str - :param column: A column name - :type str - :param mapping_database: A mapping database name (e.g. `untraceables`) - :type str - :param mapping_table: A mapping table name (e.g. `users`) - :rtype list - :return Multiple queries - """ - - queries = [] - queries.append('DROP TABLE IF EXISTS `{:s}`.`_{:s}`'.format(database, table)) - queries.append('CREATE TABLE `{0:s}`.`_{1:s}` LIKE `{0:s}`.`{1:s}`'.format(database, table)) - queries.append(_get_randomize(database, table, columns, column, mapping_database, mapping_table)) - queries.append('DROP TABLE `{:s}`.`{:s}`'.format(database, table)) - queries.append('RENAME TABLE `{0:s}`.`_{1:s}` TO `{0:s}`.`{1:s}`'.format(database, table)) - - return queries + """ + Gets the queries to randomize a table / column in a given database. + + :type str + :param database: A database name + :type str + :param table: A table name + :type tuple + :param columns: Zero or more column names (result of `SHOW COLUMNS`) + :type str + :param column: A column name + :type str + :param mapping_database: A mapping database name (e.g. `untraceables`) + :type str + :param mapping_table: A mapping table name (e.g. `users`) + :rtype list + :return Multiple queries + """ + + queries = [] + queries.append('DROP TABLE IF EXISTS `{:s}`.`_{:s}`'.format(database, table)) + queries.append('CREATE TABLE `{0:s}`.`_{1:s}` LIKE `{0:s}`.`{1:s}`'.format(database, table)) + queries.append(_get_randomize(database, table, columns, column, mapping_database, mapping_table)) + queries.append('DROP TABLE `{:s}`.`{:s}`'.format(database, table)) + queries.append('RENAME TABLE `{0:s}`.`_{1:s}` TO `{0:s}`.`{1:s}`'.format(database, table)) + + return queries def _get_randomize(database, table, columns, column, mapping_database, mapping_table): - """ - Gets the query to randomize a table / column in a given database. - - INSERT INTO ... SELECT FROM ... part. - - :type str - :param database: A database name - :type str - :param table: A table name - :type tuple - :param columns: Zero or more column names (result of `SHOW COLUMNS`) - :type str - :param column: A column name - :type str - :param mapping_database: A mapping database name (e.g. `untraceables`) - :type str - :param mapping_table: A mapping table name (e.g. `users`) - :rtype str - :return A query - """ - - query = [] - query.append('INSERT INTO `{:s}`.`_{:s}`'.format(database, table)) - query.append('SELECT') - - select = [] - for c in columns: - if c['Field'] == column: - select.append('`t2`.`{:s}`'.format(untraceables.MAPPING_ID_FIELD)) - else: - select.append('`t1`.`{:s}`'.format(c['Field'])) - query.append(', '.join(select)) - - query.append('FROM `{:s}`.`{:s}` `t1`'.format(database, table)) - query.append('LEFT JOIN `{:s}`.`{:s}` `t2` ON `t2`.`id` = `t1`.`{:s}`'.format(mapping_database, - mapping_table, - column)) - - return ' '.join(query) + """ + Gets the query to randomize a table / column in a given database. + + INSERT INTO ... SELECT FROM ... part. + + :type str + :param database: A database name + :type str + :param table: A table name + :type tuple + :param columns: Zero or more column names (result of `SHOW COLUMNS`) + :type str + :param column: A column name + :type str + :param mapping_database: A mapping database name (e.g. `untraceables`) + :type str + :param mapping_table: A mapping table name (e.g. `users`) + :rtype str + :return A query + """ + + query = [] + query.append('INSERT INTO `{:s}`.`_{:s}`'.format(database, table)) + query.append('SELECT') + + select = [] + for c in columns: + if c['Field'] == column: + select.append('`t2`.`{:s}`'.format(untraceables.MAPPING_ID_FIELD)) + else: + select.append('`t1`.`{:s}`'.format(c['Field'])) + query.append(', '.join(select)) + + query.append('FROM `{:s}`.`{:s}` `t1`'.format(database, table)) + query.append('LEFT JOIN `{:s}`.`{:s}` `t2` ON `t2`.`id` = `t1`.`{:s}`'.format(mapping_database, + mapping_table, + column)) + + return ' '.join(query) diff --git a/untraceables/utilities/validation.py b/untraceables/utilities/validation.py index f1d0660..85f2516 100644 --- a/untraceables/utilities/validation.py +++ b/untraceables/utilities/validation.py @@ -6,15 +6,15 @@ def check_max_ids(max_id, mapping_max_id): - """ - Checks that the maximum ID of the mapping table is sufficient. + """ + Checks that the maximum ID of the mapping table is sufficient. - :type long - :param max_id: The maximum id - :type long - :param mapping_max_id: The maximum id of the mapping table - :rtype bool - :return Failure - """ + :type long + :param max_id: The maximum id + :type long + :param mapping_max_id: The maximum id of the mapping table + :rtype bool + :return Failure + """ - return not mapping_max_id or mapping_max_id < max_id + return mapping_max_id is None or (max_id is not None and mapping_max_id < max_id)