diff --git a/flattentool/input.py b/flattentool/input.py index b8a659e..c055eae 100644 --- a/flattentool/input.py +++ b/flattentool/input.py @@ -1150,7 +1150,7 @@ def to_list(self): def temporarydicts_to_lists(nested_dict): - """ Recursively transforms TemporaryDicts to lists inplace. """ + """Recursively transforms TemporaryDicts to lists inplace.""" for key, value in nested_dict.items(): if isinstance(value, Cell): continue diff --git a/flattentool/json_input.py b/flattentool/json_input.py index e1cc6a1..dff38b4 100644 --- a/flattentool/json_input.py +++ b/flattentool/json_input.py @@ -87,12 +87,12 @@ def dicts_to_list_of_dicts(lists_of_dicts_paths_set, xml_dict, path=()): def list_dict_consistency(xml_dict): """ - For use with XML files opened with xmltodict. + For use with XML files opened with xmltodict. - If there is only one tag, xmltodict produces a dict. If there are - multiple, xmltodict produces a list of dicts. This functions replaces - dicts with lists of dicts, if there exists a list of dicts for the same - path elsewhere in the file. + If there is only one tag, xmltodict produces a dict. If there are + multiple, xmltodict produces a list of dicts. This functions replaces + dicts with lists of dicts, if there exists a list of dicts for the same + path elsewhere in the file. """ lists_of_dicts_paths_set = set(lists_of_dicts_paths(xml_dict)) dicts_to_list_of_dicts(lists_of_dicts_paths_set, xml_dict) @@ -214,7 +214,9 @@ def __init__( if self.xml: with codecs.open(json_filename, "rb") as xml_file: top_dict = xmltodict.parse( - xml_file, force_list=(root_list_path,), force_cdata=True, + xml_file, + force_list=(root_list_path,), + force_cdata=True, ) # AFAICT, this should be true for *all* XML files assert len(top_dict) == 1 @@ -468,13 +470,19 @@ def parse_json_dict( self.use_titles and parent_name + key in self.rollup and self.schema_parser.sub_sheet_titles.get( - (parent_name, key,) + ( + parent_name, + key, + ) ) in self.schema_parser.sub_sheets ): relevant_subsheet = self.schema_parser.sub_sheets.get( self.schema_parser.sub_sheet_titles.get( - (parent_name, key,) + ( + parent_name, + key, + ) ) ) if relevant_subsheet is not None: @@ -546,10 +554,17 @@ def parse_json_dict( if ( self.use_titles and self.schema_parser - and (parent_name, key,) in self.schema_parser.sub_sheet_titles + and ( + parent_name, + key, + ) + in self.schema_parser.sub_sheet_titles ): sub_sheet_name = self.schema_parser.sub_sheet_titles[ - (parent_name, key,) + ( + parent_name, + key, + ) ] else: sub_sheet_name = make_sub_sheet_name( diff --git a/flattentool/output.py b/flattentool/output.py index 9ad275e..3ba6561 100644 --- a/flattentool/output.py +++ b/flattentool/output.py @@ -105,7 +105,7 @@ def open(self): self.workbook = OpenDocumentSpreadsheet() def _make_cell(self, value): - """ Util for creating an ods cell """ + """Util for creating an ods cell""" if value: try: diff --git a/flattentool/schema.py b/flattentool/schema.py index 0a4bf10..b572501 100644 --- a/flattentool/schema.py +++ b/flattentool/schema.py @@ -315,7 +315,10 @@ def parse_schema_dict( path_separator=":", ) self.sub_sheet_titles[ - (parent_path, property_name,) + ( + parent_path, + property_name, + ) ] = sub_sheet_name else: sub_sheet_name = make_sub_sheet_name( diff --git a/flattentool/tests/test_end_to_end.py b/flattentool/tests/test_end_to_end.py index dbfa833..b162472 100644 --- a/flattentool/tests/test_end_to_end.py +++ b/flattentool/tests/test_end_to_end.py @@ -182,7 +182,15 @@ def test_merging_cols(): * Columns with the same name result in the first being overwritten and not appearing in the cell source map """ - sheets = [{"name": "main", "headings": ["int", "int"], "rows": [[1, 2],]}] + sheets = [ + { + "name": "main", + "headings": ["int", "int"], + "rows": [ + [1, 2], + ], + } + ] # XXX We don't correctly get a list of lists here, just [OrderedDict([(u'int', 2)])] expected_result = [[OrderedDict([("int", 2)])]] # XXX Fails to keep the source map to cell B2 because the value is lost early on in @@ -193,7 +201,11 @@ def test_merging_cols(): ("main/0", [("main", 2)]), ] ) - expected_heading_source_map = OrderedDict([("main/int", [("main", "int")]),]) + expected_heading_source_map = OrderedDict( + [ + ("main/int", [("main", "int")]), + ] + ) expected = (expected_result, expected_cell_source_map, expected_heading_source_map) assert expected == run(sheets, source_maps=True) @@ -206,7 +218,9 @@ def test_merging_cols(): { "name": "main", "headings": ["name", "address/house"], - "rows": [["James", "15"],], + "rows": [ + ["James", "15"], + ], }, ] @@ -243,7 +257,9 @@ def test_merging_cols(): "definitions": { "Address": { "type": "object", - "properties": {"house": {"type": "string"},}, + "properties": { + "house": {"type": "string"}, + }, } }, "type": "object", @@ -266,7 +282,9 @@ def test_merging_cols(): "Address": { "type": "object", "title": "Address Title", - "properties": {"house": {"type": "string"},}, + "properties": { + "house": {"type": "string"}, + }, } }, "type": "object", @@ -285,7 +303,9 @@ def test_merging_cols(): { "name": "main", "headings": [" NAmE TiTLe ", " ADDresS TiTLe : HOusE TiTLe "], - "rows": [["James", "15"],], + "rows": [ + ["James", "15"], + ], }, ], { @@ -296,13 +316,19 @@ def test_merging_cols(): "type": "object", "title": "Address Title", "properties": { - "house": {"type": "string", "title": "House Title",}, + "house": { + "type": "string", + "title": "House Title", + }, }, } }, "type": "object", "properties": { - "name": {"type": "string", "title": "Name Title",}, + "name": { + "type": "string", + "title": "Name Title", + }, "address": {"$ref": "#/definitions/Address"}, }, }, @@ -352,14 +378,30 @@ def test_dict( test_list_of_dicts_data_result = [ - OrderedDict([("name", "James"), ("address", [OrderedDict([("house", "15"),])])]) + OrderedDict( + [ + ("name", "James"), + ( + "address", + [ + OrderedDict( + [ + ("house", "15"), + ] + ) + ], + ), + ] + ) ] test_list_of_dicts_data_sheets = [ { "name": "main", "headings": ["name", "address/0/house"], - "rows": [["James", "15"],], + "rows": [ + ["James", "15"], + ], }, ] @@ -387,13 +429,23 @@ def test_dict( "Address": { "type": "object", "title": "Address Item Title", - "properties": {"house": {"type": "string", "title": "House Title",},}, + "properties": { + "house": { + "type": "string", + "title": "House Title", + }, + }, }, }, "properties": { - "name": {"type": "string", "title": "Name Title",}, + "name": { + "type": "string", + "title": "Name Title", + }, "address": { - "items": {"$ref": "#/definitions/Address",}, + "items": { + "$ref": "#/definitions/Address", + }, "type": "array", "title": "Address Title", }, @@ -419,13 +471,21 @@ def test_dict( "Address": { "type": "object", "title": "Address Item Title", - "properties": {"house": {"type": "string",},}, + "properties": { + "house": { + "type": "string", + }, + }, }, }, "properties": { - "name": {"type": "string",}, + "name": { + "type": "string", + }, "address": { - "items": {"$ref": "#/definitions/Address",}, + "items": { + "$ref": "#/definitions/Address", + }, "type": "array", }, }, @@ -448,7 +508,9 @@ def test_dict( { "name": "main", "headings": [" NAmE TiTLe ", " ADDresS TiTLe : 0 : HOusE TiTLe "], - "rows": [["James", "15"],], + "rows": [ + ["James", "15"], + ], }, ], test_list_of_dicts_data_schema_with_titles, @@ -498,7 +560,17 @@ def test_list_of_dicts( [ ("id", "person1"), ("name", "James"), - ("address", [OrderedDict([("id", "address1"), ("house", "15"),])]), + ( + "address", + [ + OrderedDict( + [ + ("id", "address1"), + ("house", "15"), + ] + ) + ], + ), ] ) ] @@ -507,7 +579,9 @@ def test_list_of_dicts( { "name": "main", "headings": ["id", "name", "address/0/id", "address/0/house"], - "rows": [["person1", "James", "address1", "15"],], + "rows": [ + ["person1", "James", "address1", "15"], + ], }, ] @@ -540,16 +614,30 @@ def test_list_of_dicts( "type": "object", "title": "Address Item Title", "properties": { - "house": {"type": "string", "title": "House Title",}, - "id": {"type": "string", "title": "Identifier",}, + "house": { + "type": "string", + "title": "House Title", + }, + "id": { + "type": "string", + "title": "Identifier", + }, }, }, }, "properties": { - "id": {"type": "string", "title": "Identifier",}, - "name": {"type": "string", "title": "Name Title",}, + "id": { + "type": "string", + "title": "Identifier", + }, + "name": { + "type": "string", + "title": "Name Title", + }, "address": { - "items": {"$ref": "#/definitions/Address",}, + "items": { + "$ref": "#/definitions/Address", + }, "type": "array", "title": "Address Title", }, @@ -575,16 +663,26 @@ def test_list_of_dicts( "Address": { "type": "object", "properties": { - "house": {"type": "string",}, - "id": {"type": "string",}, + "house": { + "type": "string", + }, + "id": { + "type": "string", + }, }, }, }, "properties": { - "id": {"type": "string",}, - "name": {"type": "string",}, + "id": { + "type": "string", + }, + "name": { + "type": "string", + }, "address": { - "items": {"$ref": "#/definitions/Address",}, + "items": { + "$ref": "#/definitions/Address", + }, "type": "array", }, }, @@ -612,7 +710,9 @@ def test_list_of_dicts( " ADDresS TiTLe : 0 : IDENtifiER ", " ADDresS TiTLe : 0 : HOusE TiTLe ", ], - "rows": [["person1", "James", "address1", "15"],], + "rows": [ + ["person1", "James", "address1", "15"], + ], }, ], test_list_of_dicts_with_ids_data_schema_with_titles, @@ -671,7 +771,10 @@ def test_list_of_dicts_with_ids( { "name": "main", "headings": ["id", "name"], - "rows": [["PERSON-james", "James"], ["PERSON-bob", "Bob"],], + "rows": [ + ["PERSON-james", "James"], + ["PERSON-bob", "Bob"], + ], }, { "name": "addresses", @@ -917,7 +1020,12 @@ def test_list_of_dicts_with_ids( ], {"main:2": 2, "main:3": 2, "main:5": 2, "main:4": 2}, ), - ["main:address/0/house", "main:address/0/town", "main:id", "main:name",], + [ + "main:address/0/house", + "main:address/0/town", + "main:id", + "main:name", + ], ), ) @@ -947,8 +1055,18 @@ def test_arrangement( ( "address", [ - OrderedDict([("house", "1"), ("town", "London"),]), - OrderedDict([("house", "2"), ("town", "Birmingham"),]), + OrderedDict( + [ + ("house", "1"), + ("town", "London"), + ] + ), + OrderedDict( + [ + ("house", "2"), + ("town", "Birmingham"), + ] + ), ], ), ] @@ -960,8 +1078,18 @@ def test_arrangement( ( "address", [ - OrderedDict([("house", "3"), ("town", "Leeds"),]), - OrderedDict([("house", "4"), ("town", "Manchester"),]), + OrderedDict( + [ + ("house", "3"), + ("town", "Leeds"), + ] + ), + OrderedDict( + [ + ("house", "4"), + ("town", "Manchester"), + ] + ), ], ), ] @@ -1027,7 +1155,11 @@ def run(sheets, schema=None, source_maps=False): parser.parse() spreadsheet_input.parser = parser else: - spreadsheet_input = HeadingListInput(input_sheets, input_headings, root_id="",) + spreadsheet_input = HeadingListInput( + input_sheets, + input_headings, + root_id="", + ) spreadsheet_input.read_sheets() if source_maps: ( diff --git a/flattentool/tests/test_headings.py b/flattentool/tests/test_headings.py index 9d82209..cab2945 100644 --- a/flattentool/tests/test_headings.py +++ b/flattentool/tests/test_headings.py @@ -21,7 +21,9 @@ ["a", "b", "c", "b", "c", "c", "d", "d", "d", "d"], [ # Check for warnings even with empty cells - [1,], + [ + 1, + ], ], [ 'Duplicate heading "b" found, ignoring the data in column B (sheet: "main").', @@ -39,7 +41,13 @@ def test_duplicate_headings_give_warning( headings, rows, expected_warnings, expected_result ): - sheets = [{"name": "main", "headings": headings, "rows": rows,}] + sheets = [ + { + "name": "main", + "headings": headings, + "rows": rows, + } + ] with pytest.warns(DataErrorWarning) as type_warnings: result = run(sheets) # Check that only one warning was raised @@ -91,7 +99,11 @@ def run(sheets, schema=None, source_maps=False): parser.parse() spreadsheet_input.parser = parser else: - spreadsheet_input = HeadingListInput(input_sheets, input_headings, root_id="",) + spreadsheet_input = HeadingListInput( + input_sheets, + input_headings, + root_id="", + ) spreadsheet_input.read_sheets() if source_maps: ( diff --git a/flattentool/tests/test_input_SpreadsheetInput.py b/flattentool/tests/test_input_SpreadsheetInput.py index 61ca2e5..53af1b4 100644 --- a/flattentool/tests/test_input_SpreadsheetInput.py +++ b/flattentool/tests/test_input_SpreadsheetInput.py @@ -261,8 +261,8 @@ def test_ods_input_integer2(self): assert list(odsinput.sub_sheet_names) == ["Sheet1"] def test_xlsx_input_formula(self): - """ When a formula is present, we should use the value, rather than the - formula itself. """ + """When a formula is present, we should use the value, rather than the + formula itself.""" xlsxinput = XLSXInput(input_name="flattentool/tests/fixtures/xlsx/formula.xlsx") @@ -279,7 +279,7 @@ def test_xlsx_input_formula(self): ] def test_bad_xlsx(self): - """ XLSX file that is not a XLSX""" + """XLSX file that is not a XLSX""" xlsxinput = XLSXInput(input_name="flattentool/tests/fixtures/xlsx/file.xlsx") @@ -295,8 +295,8 @@ def test_bad_xlsx(self): assert False, "No Exception Raised" def test_ods_input_formula(self): - """ When a formula is present, we should use the value, rather than the - formula itself. """ + """When a formula is present, we should use the value, rather than the + formula itself.""" odsinput = ODSInput(input_name="flattentool/tests/fixtures/ods/formula.ods") diff --git a/flattentool/tests/test_input_SpreadsheetInput_unflatten.py b/flattentool/tests/test_input_SpreadsheetInput_unflatten.py index 0d13dce..8bcaa01 100644 --- a/flattentool/tests/test_input_SpreadsheetInput_unflatten.py +++ b/flattentool/tests/test_input_SpreadsheetInput_unflatten.py @@ -88,7 +88,14 @@ def inject_root_id(root_id, d): ), ( "Nested", - [{"ROOT_ID": "1", "id": 2, "testO/testB": 3, "testO/testC": 4,}], + [ + { + "ROOT_ID": "1", + "id": 2, + "testO/testB": 3, + "testO/testC": 4, + } + ], [{"ROOT_ID": "1", "id": 2, "testO": {"testB": 3, "testC": 4}}], [], True, @@ -105,15 +112,32 @@ def inject_root_id(root_id, d): ( "Single item array", [{"ROOT_ID": "1", "id": 2, "testL/0/id": 3, "testL/0/testB": 4}], - [{"ROOT_ID": "1", "id": 2, "testL": [{"id": 3, "testB": 4}],}], + [ + { + "ROOT_ID": "1", + "id": 2, + "testL": [{"id": 3, "testB": 4}], + } + ], [], False, True, ), ( "Single item array without parent ID", - [{"ROOT_ID": "1", "testL/0/id": "2", "testL/0/testB": "3",}], - [{"ROOT_ID": "1", "testL": [{"id": "2", "testB": "3"}],}], + [ + { + "ROOT_ID": "1", + "testL/0/id": "2", + "testL/0/testB": "3", + } + ], + [ + { + "ROOT_ID": "1", + "testL": [{"id": "2", "testB": "3"}], + } + ], [], False, True, @@ -160,10 +184,23 @@ def inject_root_id(root_id, d): "Mismatch of object/array for field not in schema", [ OrderedDict( - [("ROOT_ID", 1), ("id", 2), ("newtest/a", 3), ("newtest/0/a", 4),] + [ + ("ROOT_ID", 1), + ("id", 2), + ("newtest/a", 3), + ("newtest/0/a", 4), + ] ) ], - [{"ROOT_ID": 1, "id": 2, "newtest": {"a": 3,}}], + [ + { + "ROOT_ID": 1, + "id": 2, + "newtest": { + "a": 3, + }, + } + ], [ "Column newtest/0/a has been ignored, because it treats newtest as an array, but another column does not." ], @@ -176,7 +213,12 @@ def inject_root_id(root_id, d): "Mismatch of array/object for field not in schema", [ OrderedDict( - [("ROOT_ID", 1), ("id", 2), ("newtest/0/a", 4), ("newtest/a", 3),] + [ + ("ROOT_ID", 1), + ("id", 2), + ("newtest/0/a", 4), + ("newtest/a", 3), + ] ) ], [{"ROOT_ID": 1, "id": 2, "newtest": [{"a": 4}]}], @@ -190,7 +232,16 @@ def inject_root_id(root_id, d): # Now one of the columns is ignored ( "str / array mixing", - [OrderedDict([("ROOT_ID", 1), ("id", 2), ("newtest", 3), ("newtest/0/a", 4),])], + [ + OrderedDict( + [ + ("ROOT_ID", 1), + ("id", 2), + ("newtest", 3), + ("newtest/0/a", 4), + ] + ) + ], [{"ROOT_ID": 1, "id": 2, "newtest": 3}], [ "Column newtest/0/a has been ignored, because it treats newtest as an array, but another column does not." @@ -200,7 +251,16 @@ def inject_root_id(root_id, d): ), ( "str / object mixing", - [OrderedDict([("ROOT_ID", 1), ("id", 2), ("newtest", 3), ("newtest/a", 4),])], + [ + OrderedDict( + [ + ("ROOT_ID", 1), + ("id", 2), + ("newtest", 3), + ("newtest/a", 4), + ] + ) + ], [{"ROOT_ID": 1, "id": 2, "newtest": 3}], [ "Column newtest/a has been ignored, because it treats newtest as an object, but another column does not." @@ -229,7 +289,16 @@ def inject_root_id(root_id, d): ), ( "object / str mixing", - [OrderedDict([("ROOT_ID", 1), ("id", 2), ("newtest/a", 3), ("newtest", 4),])], + [ + OrderedDict( + [ + ("ROOT_ID", 1), + ("id", 2), + ("newtest/a", 3), + ("newtest", 4), + ] + ) + ], [{"ROOT_ID": 1, "id": 2, "newtest": {"a": 3}}], [ "Column newtest has been ignored, because another column treats it as an array or object" @@ -240,10 +309,32 @@ def inject_root_id(root_id, d): ( "Mismatch of object/array for field not in schema (multiline)", [ - OrderedDict([("ROOT_ID", 1), ("id", 2), ("nest/newtest/a", 3),]), - OrderedDict([("ROOT_ID", 1), ("id", 2), ("nest/newtest/0/a", 4),]), + OrderedDict( + [ + ("ROOT_ID", 1), + ("id", 2), + ("nest/newtest/a", 3), + ] + ), + OrderedDict( + [ + ("ROOT_ID", 1), + ("id", 2), + ("nest/newtest/0/a", 4), + ] + ), + ], + [ + { + "ROOT_ID": 1, + "id": 2, + "nest": { + "newtest": { + "a": 3, + } + }, + } ], - [{"ROOT_ID": 1, "id": 2, "nest": {"newtest": {"a": 3,}}}], [ "Column nest/newtest/0/a has been ignored, because it treats newtest as an array, but another column does not" ], @@ -255,8 +346,20 @@ def inject_root_id(root_id, d): ( "Mismatch of array/object for field not in schema (multiline)", [ - OrderedDict([("ROOT_ID", 1), ("id", 2), ("newtest/0/a", 4),]), - OrderedDict([("ROOT_ID", 1), ("id", 2), ("newtest/a", 3),]), + OrderedDict( + [ + ("ROOT_ID", 1), + ("id", 2), + ("newtest/0/a", 4), + ] + ), + OrderedDict( + [ + ("ROOT_ID", 1), + ("id", 2), + ("newtest/a", 3), + ] + ), ], [{"ROOT_ID": 1, "id": 2, "newtest": [{"a": 4}]}], [ @@ -270,7 +373,13 @@ def inject_root_id(root_id, d): ( "str / array mixing multiline", [ - OrderedDict([("ROOT_ID", 1), ("id", 2), ("nest/newtest", 3),]), + OrderedDict( + [ + ("ROOT_ID", 1), + ("id", 2), + ("nest/newtest", 3), + ] + ), OrderedDict( [ ("ROOT_ID", 1), @@ -292,8 +401,20 @@ def inject_root_id(root_id, d): "array / str mixing multiline", # same as above, but with rows switched [ - OrderedDict([("ROOT_ID", 1), ("id", 2), ("nest/newtest/0/a", 4),]), - OrderedDict([("ROOT_ID", 1), ("id", 2), ("nest/newtest", 3),]), + OrderedDict( + [ + ("ROOT_ID", 1), + ("id", 2), + ("nest/newtest/0/a", 4), + ] + ), + OrderedDict( + [ + ("ROOT_ID", 1), + ("id", 2), + ("nest/newtest", 3), + ] + ), ], [{"ROOT_ID": 1, "id": 2, "nest": {"newtest": [{"a": 4}]}}], [ @@ -306,9 +427,20 @@ def inject_root_id(root_id, d): ( "str / object mixing multiline", [ - OrderedDict([("ROOT_ID", 1), ("id", 2), ("newtest", 3),]), OrderedDict( - [("ROOT_ID", 1), ("id", 2), ("newtest/a", 4), ("newtest/b", 5),] + [ + ("ROOT_ID", 1), + ("id", 2), + ("newtest", 3), + ] + ), + OrderedDict( + [ + ("ROOT_ID", 1), + ("id", 2), + ("newtest/a", 4), + ("newtest/b", 5), + ] ), ], [{"ROOT_ID": 1, "id": 2, "newtest": 3}], @@ -322,8 +454,20 @@ def inject_root_id(root_id, d): ( "object / str mixing multiline", [ - OrderedDict([("ROOT_ID", 1), ("id", 2), ("newtest/a", 4),]), - OrderedDict([("ROOT_ID", 1), ("id", 2), ("newtest", 3),]), + OrderedDict( + [ + ("ROOT_ID", 1), + ("id", 2), + ("newtest/a", 4), + ] + ), + OrderedDict( + [ + ("ROOT_ID", 1), + ("id", 2), + ("newtest", 3), + ] + ), ], [{"ROOT_ID": 1, "id": 2, "newtest": {"a": 4}}], [ @@ -338,7 +482,14 @@ def inject_root_id(root_id, d): # (Cove will do this automatically). ( "Root ID is missing", - [OrderedDict([("id", 2), ("testA", 3),])], + [ + OrderedDict( + [ + ("id", 2), + ("testA", 3), + ] + ) + ], [{"id": 2, "testA": 3}], [], False, @@ -360,7 +511,13 @@ def inject_root_id(root_id, d): ] ) ], - [{"2.2": "B", "2.3": "C", "False": "D",}], + [ + { + "2.2": "B", + "2.3": "C", + "False": "D", + } + ], [ 'Column "1" has been ignored because it is a number.', 'Column "2" has been ignored because it is a number.', @@ -377,8 +534,20 @@ def inject_root_id(root_id, d): testdata_pointer = [ ( "Single item array without json numbering", - [{"ROOT_ID": "1", "testR/id": "2", "testR/testB": "3", "testR/testX": "3",}], - [{"ROOT_ID": "1", "testR": [{"id": "2", "testB": "3", "testX": "3"}],}], + [ + { + "ROOT_ID": "1", + "testR/id": "2", + "testR/testB": "3", + "testR/testX": "3", + } + ], + [ + { + "ROOT_ID": "1", + "testR": [{"id": "2", "testB": "3", "testX": "3"}], + } + ], [], ), ( @@ -415,25 +584,49 @@ def inject_root_id(root_id, d): def create_schema(root_id): schema = { "properties": { - "id": {"title": "Identifier", "type": "integer",}, - "testA": {"title": "A title", "type": "integer",}, - "testDateTime": {"type": "string", "format": "date-time",}, + "id": { + "title": "Identifier", + "type": "integer", + }, + "testA": { + "title": "A title", + "type": "integer", + }, + "testDateTime": { + "type": "string", + "format": "date-time", + }, "testDate": {"type": "string", "format": "date"}, "testB": { "title": "B title", "type": "object", "properties": { - "id": {"title": "Identifier", "type": "integer",}, - "testC": {"title": "C title", "type": "integer",}, - "testD": {"title": "D title", "type": "integer",}, + "id": { + "title": "Identifier", + "type": "integer", + }, + "testC": { + "title": "C title", + "type": "integer", + }, + "testD": { + "title": "D title", + "type": "integer", + }, "subField": { "title": "Sub title", "type": "array", "items": { "type": "object", "properties": { - "id": {"title": "Identifier", "type": "integer",}, - "testE": {"title": "E title", "type": "integer",}, + "id": { + "title": "Identifier", + "type": "integer", + }, + "testE": { + "title": "E title", + "type": "integer", + }, }, }, }, @@ -445,17 +638,32 @@ def create_schema(root_id): "items": { "type": "object", "properties": { - "id": {"title": "Identifier", "type": "string",}, - "testB": {"title": "B title", "type": "string",}, - "testC": {"title": "C title", "type": "string",}, + "id": { + "title": "Identifier", + "type": "string", + }, + "testB": { + "title": "B title", + "type": "string", + }, + "testC": { + "title": "C title", + "type": "string", + }, "testNest": { "title": "Nest title", "type": "array", "items": { "type": "object", "properties": { - "id": {"title": "Identifier", "type": "string",}, - "testD": {"title": "D title", "type": "string",}, + "id": { + "title": "Identifier", + "type": "string", + }, + "testD": { + "title": "D title", + "type": "string", + }, }, }, }, @@ -463,8 +671,14 @@ def create_schema(root_id): "title": "NestObj title", "type": "object", "properties": { - "id": {"title": "Identifier", "type": "string",}, - "testD": {"title": "D title", "type": "string",}, + "id": { + "title": "Identifier", + "type": "string", + }, + "testD": { + "title": "D title", + "type": "string", + }, }, }, }, @@ -484,8 +698,14 @@ def create_schema(root_id): # integer does not work, as testB:integer is not # in the rollUp }, - "testB": {"title": "B title", "type": "string",}, - "testC": {"title": "C title", "type": "string",}, + "testB": { + "title": "B title", + "type": "string", + }, + "testC": { + "title": "C title", + "type": "string", + }, "testSA": { "title": "SA title", "type": "array", @@ -494,7 +714,10 @@ def create_schema(root_id): }, }, }, - "testU": {"title": UNICODE_TEST_STRING, "type": "string",}, + "testU": { + "title": UNICODE_TEST_STRING, + "type": "string", + }, "testSA": { "title": "SA title", "type": "array", @@ -579,14 +802,25 @@ def create_schema(root_id): "R title:B title": 4, } ], - [{"ROOT_ID": 1, "id": 2, "testR": [{"id": "3", "testB": "4"}],}], + [ + { + "ROOT_ID": 1, + "id": 2, + "testR": [{"id": "3", "testB": "4"}], + } + ], [], False, ), ( "Single item array without parent ID", [{"ROOT_ID_TITLE": "1", "R title:Identifier": "2", "R title:B title": "3"}], - [{"ROOT_ID": "1", "testR": [{"id": "2", "testB": "3"}],}], + [ + { + "ROOT_ID": "1", + "testR": [{"id": "2", "testB": "3"}], + } + ], [], False, ), @@ -603,7 +837,13 @@ def create_schema(root_id): "R title:C title": 4, } ], - [{"ROOT_ID": 1, "id": 2, "testR": [{"id": "3", "testC": "4"}],}], + [ + { + "ROOT_ID": 1, + "id": 2, + "testR": [{"id": "3", "testC": "4"}], + } + ], [], False, ), @@ -617,7 +857,13 @@ def create_schema(root_id): "R title:Not in schema": 4, } ], - [{"ROOT_ID": 1, "id": 2, "testR": [{"testC": "3", "Not in schema": 4}],}], + [ + { + "ROOT_ID": 1, + "id": 2, + "testR": [{"testC": "3", "Not in schema": 4}], + } + ], [], False, ), @@ -685,29 +931,85 @@ def create_schema(root_id): ), ( "Test arrays of strings (1 item)", - [{"ROOT_ID_TITLE": 1, "Identifier": 2, "SA title": "a",}], - [{"ROOT_ID": 1, "id": 2, "testSA": ["a"],}], + [ + { + "ROOT_ID_TITLE": 1, + "Identifier": 2, + "SA title": "a", + } + ], + [ + { + "ROOT_ID": 1, + "id": 2, + "testSA": ["a"], + } + ], [], True, ), ( "Test arrays of strings (2 items)", - [{"ROOT_ID_TITLE": 1, "Identifier": 2, "SA title": "a;b",}], - [{"ROOT_ID": 1, "id": 2, "testSA": ["a", "b"],}], + [ + { + "ROOT_ID_TITLE": 1, + "Identifier": 2, + "SA title": "a;b", + } + ], + [ + { + "ROOT_ID": 1, + "id": 2, + "testSA": ["a", "b"], + } + ], [], True, ), ( "Test arrays of strings within an object array (1 item)", - [{"ROOT_ID_TITLE": 1, "Identifier": 2, "R title:SA title": "a",}], - [{"ROOT_ID": 1, "id": 2, "testR": [{"testSA": ["a"],}]}], + [ + { + "ROOT_ID_TITLE": 1, + "Identifier": 2, + "R title:SA title": "a", + } + ], + [ + { + "ROOT_ID": 1, + "id": 2, + "testR": [ + { + "testSA": ["a"], + } + ], + } + ], [], False, ), ( "Test arrays of strings within an object array (2 items)", - [{"ROOT_ID_TITLE": 1, "Identifier": 2, "R title:SA title": "a;b",}], - [{"ROOT_ID": 1, "id": 2, "testR": [{"testSA": ["a", "b"],}]}], + [ + { + "ROOT_ID_TITLE": 1, + "Identifier": 2, + "R title:SA title": "a;b", + } + ], + [ + { + "ROOT_ID": 1, + "id": 2, + "testR": [ + { + "testSA": ["a", "b"], + } + ], + } + ], [], False, ), diff --git a/flattentool/tests/test_input_SpreadsheetInput_unflatten_mulitplesheets.py b/flattentool/tests/test_input_SpreadsheetInput_unflatten_mulitplesheets.py index ba48d01..9c845ee 100644 --- a/flattentool/tests/test_input_SpreadsheetInput_unflatten_mulitplesheets.py +++ b/flattentool/tests/test_input_SpreadsheetInput_unflatten_mulitplesheets.py @@ -23,14 +23,38 @@ ( "Basic sub sheet", { - "custom_main": [{"ROOT_ID": 1, "id": 2,}, {"ROOT_ID": 1, "id": 3,}], + "custom_main": [ + { + "ROOT_ID": 1, + "id": 2, + }, + { + "ROOT_ID": 1, + "id": 3, + }, + ], "testArr": [ - {"ROOT_ID": 1, "id": 2, "testArr/0/testC": "3",}, - {"ROOT_ID": 1, "id": 2, "testArr/0/testC": "4",}, + { + "ROOT_ID": 1, + "id": 2, + "testArr/0/testC": "3", + }, + { + "ROOT_ID": 1, + "id": 2, + "testArr/0/testC": "4", + }, ], }, [ - {"ROOT_ID": 1, "id": 2, "testArr": [{"testC": "3"}, {"testC": "4"},]}, + { + "ROOT_ID": 1, + "id": 2, + "testArr": [ + {"testC": "3"}, + {"testC": "4"}, + ], + }, {"ROOT_ID": 1, "id": 3}, ], [], @@ -39,11 +63,23 @@ ( "Nested sub sheet (with id)", { - "custom_main": [{"ROOT_ID": 1, "id": 2, "testB/id": 3, "testB/testC": 4,}], + "custom_main": [ + { + "ROOT_ID": 1, + "id": 2, + "testB/id": 3, + "testB/testC": 4, + } + ], "tes_subField": [ # It used to be necessary to supply testA/id in this # situation, but now it's optional - {"ROOT_ID": 1, "id": 2, "testB/id": 3, "testB/subField/0/testD": 5,} + { + "ROOT_ID": 1, + "id": 2, + "testB/id": 3, + "testB/subField/0/testD": 5, + } ], }, [ @@ -59,11 +95,22 @@ ( "Nested sub sheet (without id)", { - "custom_main": [{"ROOT_ID": 1, "id": 2, "testB/id": 3, "testB/testC": 4,}], + "custom_main": [ + { + "ROOT_ID": 1, + "id": 2, + "testB/id": 3, + "testB/testC": 4, + } + ], "sub": [ # It used to be necessary to supply testA/id in this # situation, but now it's optional - {"ROOT_ID": 1, "id": 2, "testB/subField/0/testD": 5,} + { + "ROOT_ID": 1, + "id": 2, + "testB/subField/0/testD": 5, + } ], }, [ @@ -83,8 +130,18 @@ ( "custom_main", [ - OrderedDict([("ROOT_ID", 1), ("id", 2),]), - OrderedDict([("ROOT_ID", 1), ("id", 6),]), + OrderedDict( + [ + ("ROOT_ID", 1), + ("id", 2), + ] + ), + OrderedDict( + [ + ("ROOT_ID", 1), + ("id", 6), + ] + ), ], ), ( @@ -127,9 +184,19 @@ ( "Nested id", { - "custom_main": [{"ROOT_ID": 1, "id": 2,}], + "custom_main": [ + { + "ROOT_ID": 1, + "id": 2, + } + ], "subField": [ - {"ROOT_ID": 1, "id": 2, "subField/0/id": 3, "subField/0/testA/id": 4,} + { + "ROOT_ID": 1, + "id": 2, + "subField/0/id": 3, + "subField/0/testA/id": 4, + } ], }, [{"ROOT_ID": 1, "id": 2, "subField": [{"id": 3, "testA": {"id": 4}}]}], @@ -139,10 +206,25 @@ ( "Missing columns", { - "custom_main": [{"ROOT_ID": 1, "id": 2,}], + "custom_main": [ + { + "ROOT_ID": 1, + "id": 2, + } + ], "sub": [ - {"ROOT_ID": 1, "id": "", "subField/0/id": 3, "subField/0/testA": 4,}, - {"ROOT_ID": 1, "id": 2, "subField/0/id": 3, "subField/0/testA": 5,}, + { + "ROOT_ID": 1, + "id": "", + "subField/0/id": 3, + "subField/0/testA": 4, + }, + { + "ROOT_ID": 1, + "id": 2, + "subField/0/id": 3, + "subField/0/testA": 5, + }, ], }, [ @@ -156,7 +238,15 @@ "Unmatched id", OrderedDict( [ - ("custom_main", [{"ROOT_ID": 1, "id": 2,}]), + ( + "custom_main", + [ + { + "ROOT_ID": 1, + "id": 2, + } + ], + ), ( "sub", [ @@ -194,11 +284,25 @@ "testArr/0/id": "4", "testArr/0/testB": "5", }, - {"ROOT_ID": 6, "id": 7, "testC": 8, "testArr/0/testB": "9",}, + { + "ROOT_ID": 6, + "id": 7, + "testC": 8, + "testArr/0/testB": "9", + }, ], "testArr": [ - {"ROOT_ID": 1, "id": 2, "testArr/0/id": "4", "testArr/0/testB": "5",}, - {"ROOT_ID": 6, "id": 7, "testArr/0/testB": "9",}, + { + "ROOT_ID": 1, + "id": 2, + "testArr/0/id": "4", + "testArr/0/testB": "5", + }, + { + "ROOT_ID": 6, + "id": 7, + "testArr/0/testB": "9", + }, ], }, [ @@ -319,23 +423,48 @@ "Basic sub sheet", { "custom_main": [ - {"ROOT_ID": 1, "Identifier": 2,}, - {"ROOT_ID": 1, "Identifier": 3,}, + { + "ROOT_ID": 1, + "Identifier": 2, + }, + { + "ROOT_ID": 1, + "Identifier": 3, + }, ], "Arr title": [ - {"ROOT_ID": 1, "Identifier": 2, "Arr title:C title": "3",}, - {"ROOT_ID": 1, "Identifier": 2, "Arr title:C title": "4",}, + { + "ROOT_ID": 1, + "Identifier": 2, + "Arr title:C title": "3", + }, + { + "ROOT_ID": 1, + "Identifier": 2, + "Arr title:C title": "4", + }, ], "arr_not_in_schema": [ - {"ROOT_ID": 1, "Identifier": 2, "arr_not_in_schema/0/testD": "5",}, - {"ROOT_ID": 1, "Identifier": 2, "arr_not_in_schema/0/testD": "6",}, + { + "ROOT_ID": 1, + "Identifier": 2, + "arr_not_in_schema/0/testD": "5", + }, + { + "ROOT_ID": 1, + "Identifier": 2, + "arr_not_in_schema/0/testD": "6", + }, ], }, [ { "ROOT_ID": 1, "id": 2, - "testArr": [{"testC": "3"}, {"testC": "4"},], + "testArr": [ + {"testC": "3"}, + {"testC": "4"}, + ], "arr_not_in_schema": [{"testD": "5"}, {"testD": "6"}], }, {"ROOT_ID": 1, "id": 3}, @@ -389,7 +518,11 @@ "sub": [ # It used to be necessary to supply testA/id in this # situation, but now it's optional - {"ROOT_ID": 1, "Identifier": 2, "B title:Sub title:E title": 5,} + { + "ROOT_ID": 1, + "Identifier": 2, + "B title:Sub title:E title": 5, + } ], }, [ @@ -409,8 +542,18 @@ ( "custom_main", [ - OrderedDict([("ROOT_ID", 1), ("Identifier", 2),]), - OrderedDict([("ROOT_ID", 1), ("Identifier", 6),]), + OrderedDict( + [ + ("ROOT_ID", 1), + ("Identifier", 2), + ] + ), + OrderedDict( + [ + ("ROOT_ID", 1), + ("Identifier", 6), + ] + ), ], ), ( @@ -456,7 +599,12 @@ ( "Nested id", { - "custom_main": [{"ROOT_ID": 1, "Identifier": 2,}], + "custom_main": [ + { + "ROOT_ID": 1, + "Identifier": 2, + } + ], "Arr title": [ { "ROOT_ID": 1, @@ -473,7 +621,12 @@ ( "Missing columns", { - "custom_main": [{"ROOT_ID": 1, "Identifier": 2,}], + "custom_main": [ + { + "ROOT_ID": 1, + "Identifier": 2, + } + ], "sub": [ { "ROOT_ID": 1, @@ -500,7 +653,15 @@ "Unmatched id", OrderedDict( [ - ("custom_main", [{"ROOT_ID": 1, "Identifier": 2,}]), + ( + "custom_main", + [ + { + "ROOT_ID": 1, + "Identifier": 2, + } + ], + ), ( "sub", [ @@ -538,7 +699,12 @@ "Arr title:Identifier": 4, "Arr title:B title": 5, }, - {"ROOT_ID": 6, "Identifier": 7, "A title": 8, "Arr title:B title": 9,}, + { + "ROOT_ID": 6, + "Identifier": 7, + "A title": 8, + "Arr title:B title": 9, + }, ], "testArr": [ { @@ -547,7 +713,11 @@ "Arr title:Identifier": 4, "Arr title:B title": 5, }, - {"ROOT_ID": 6, "Identifier": 7, "Arr title:B title": 9,}, + { + "ROOT_ID": 6, + "Identifier": 7, + "Arr title:B title": 9, + }, ], }, [ @@ -619,7 +789,12 @@ { "custom_main": [], "subsheet": [ - {"ROOT_ID": "", "Identifier": "", "A title": "", "U title": "",} + { + "ROOT_ID": "", + "Identifier": "", + "A title": "", + "U title": "", + } ], }, [], diff --git a/flattentool/tests/test_json_input.py b/flattentool/tests/test_json_input.py index 3535786..72faebe 100644 --- a/flattentool/tests/test_json_input.py +++ b/flattentool/tests/test_json_input.py @@ -88,8 +88,18 @@ def test_parse_empty_json_dict(): def test_parse_basic_json_dict(): parser = JSONParser( root_json_dict=[ - OrderedDict([("a", "b"), ("c", "d"),]), - OrderedDict([("a", "e"), ("c", "f"),]), + OrderedDict( + [ + ("a", "b"), + ("c", "d"), + ] + ), + OrderedDict( + [ + ("a", "e"), + ("c", "f"), + ] + ), ] ) assert list(parser.main_sheet) == ["a", "c"] @@ -102,7 +112,14 @@ def test_parse_basic_json_dict(): def test_parse_nested_dict_json_dict(): parser = JSONParser( - root_json_dict=[OrderedDict([("a", "b"), ("c", OrderedDict([("d", "e")])),])] + root_json_dict=[ + OrderedDict( + [ + ("a", "b"), + ("c", OrderedDict([("d", "e")])), + ] + ) + ] ) assert list(parser.main_sheet) == ["a", "c/d"] assert list(parser.main_sheet.lines) == [{"a": "b", "c/d": "e"}] @@ -111,7 +128,14 @@ def test_parse_nested_dict_json_dict(): def test_parse_nested_list_json_dict(): parser = JSONParser( - root_json_dict=[OrderedDict([("a", "b"), ("c", [OrderedDict([("d", "e")])]),])] + root_json_dict=[ + OrderedDict( + [ + ("a", "b"), + ("c", [OrderedDict([("d", "e")])]), + ] + ) + ] ) assert list(parser.main_sheet) == ["a"] assert list(parser.main_sheet.lines) == [{"a": "b"}] @@ -130,7 +154,16 @@ def test_parse_array(): def test_root_list_path(): parser = JSONParser( - root_json_dict={"custom_key": [OrderedDict([("a", "b"), ("c", "d"),])]}, + root_json_dict={ + "custom_key": [ + OrderedDict( + [ + ("a", "b"), + ("c", "d"), + ] + ) + ] + }, root_list_path="custom_key", ) assert list(parser.main_sheet) == ["a", "c"] @@ -208,7 +241,12 @@ def test_parse_ids_subsheet(self): root_id="ocid", ) assert list(parser.main_sheet) == ["ocid", "id"] - assert list(parser.main_sheet.lines) == [{"ocid": 1, "id": 2,}] + assert list(parser.main_sheet.lines) == [ + { + "ocid": 1, + "id": 2, + } + ] assert listify(parser.sub_sheets) == { "testnest": [ "ocid", @@ -314,7 +352,12 @@ def test_sub_sheets(self, tmpdir, remove_empty_schema_columns): schema_parser.parse() parser = JSONParser( root_json_dict=[ - OrderedDict([("a", "b"), ("c", [OrderedDict([("d", "e")])]),]) + OrderedDict( + [ + ("a", "b"), + ("c", [OrderedDict([("d", "e")])]), + ] + ) ], schema_parser=schema_parser, remove_empty_schema_columns=remove_empty_schema_columns, @@ -344,7 +387,14 @@ def test_column_matching(self, tmpdir): schema_parser = SchemaParser(schema_filename=test_schema.strpath) schema_parser.parse() parser = JSONParser( - root_json_dict=[OrderedDict([("c", ["d"]),])], schema_parser=schema_parser, + root_json_dict=[ + OrderedDict( + [ + ("c", ["d"]), + ] + ) + ], + schema_parser=schema_parser, ) assert list(parser.main_sheet) == ["c"] assert list(parser.main_sheet.lines) == [{"c": "d"}] @@ -374,7 +424,9 @@ def test_rollup(self): parser = JSONParser( root_json_dict=[ OrderedDict( - [("testA", [OrderedDict([("testB", "1"), ("testC", "2")])]),] + [ + ("testA", [OrderedDict([("testB", "1"), ("testC", "2")])]), + ] ) ], schema_parser=schema_parser, @@ -580,7 +632,12 @@ def test_parse_ids_subsheet(self): root_id="custom", ) assert list(parser.main_sheet) == ["custom", "id"] - assert list(parser.main_sheet.lines) == [{"custom": 1, "id": 2,}] + assert list(parser.main_sheet.lines) == [ + { + "custom": 1, + "id": 2, + } + ] assert listify(parser.sub_sheets) == { "testnest": [ "custom", @@ -716,13 +773,22 @@ def test_parse_ids_subsheet(self): root_id="", ) assert list(parser.main_sheet) == ["id"] - assert list(parser.main_sheet.lines) == [{"id": 2,}] + assert list(parser.main_sheet.lines) == [ + { + "id": 2, + } + ] assert listify(parser.sub_sheets) == { "testnest": ["id", "testnest/0/id", "testnest/0/a", "testnest/0/f/g"], "tes_c": ["id", "testnest/0/id", "testnest/0/c/0/d"], } assert list(parser.sub_sheets["testnest"].lines) == [ - {"id": 2, "testnest/0/id": 3, "testnest/0/a": "b", "testnest/0/f/g": "h",}, + { + "id": 2, + "testnest/0/id": 3, + "testnest/0/a": "b", + "testnest/0/f/g": "h", + }, ] assert list(parser.sub_sheets["tes_c"].lines) == [ {"id": 2, "testnest/0/id": 3, "testnest/0/c/0/d": "e"}, diff --git a/flattentool/tests/test_output.py b/flattentool/tests/test_output.py index ea47407..2ce7c35 100644 --- a/flattentool/tests/test_output.py +++ b/flattentool/tests/test_output.py @@ -232,7 +232,9 @@ def test_utf8(tmpdir): # Check CSV assert set(tmpdir.join("release").listdir()) == set( - [tmpdir.join("release").join("release.csv"),] + [ + tmpdir.join("release").join("release.csv"), + ] ) release_csv_text = tmpdir.join("release", "release.csv").read_text(encoding="utf-8") assert release_csv_text.strip("\r\n").replace("\r", "") == "é\néαГ😼𝒞人\ncell2" diff --git a/flattentool/tests/test_schema_parser.py b/flattentool/tests/test_schema_parser.py index ffc8f1f..eecc9df 100644 --- a/flattentool/tests/test_schema_parser.py +++ b/flattentool/tests/test_schema_parser.py @@ -727,8 +727,14 @@ def test_titles_rollup(): "items": { "type": "object", "properties": { - "Btest": {"type": "string", "title": "BTitle",}, - "Ctest": {"type": "string", "title": "CTitle",}, + "Btest": { + "type": "string", + "title": "BTitle", + }, + "Ctest": { + "type": "string", + "title": "CTitle", + }, }, }, }, diff --git a/requirements_dev.txt b/requirements_dev.txt index 71ed84b..366ae58 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -8,5 +8,5 @@ sphinx sphinx_rtd_theme isort flake8 -black==19.10b0 +black==22.3.0 transifex-client diff --git a/setup.py b/setup.py index d9af199..dd027ae 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,10 @@ def run(self): description="Tools for generating CSV and other flat versions of the structured data", install_requires=install_requires, extras_require={"HTTP": ["requests"]}, - cmdclass={"install": InstallWithCompile, "develop": DevelopWithCompile,}, + cmdclass={ + "install": InstallWithCompile, + "develop": DevelopWithCompile, + }, package_data={"flattentool": ["locale/*/*/*.mo", "locale/*/*/*.po"]}, setup_requires=["babel"], )