From 6906dc9da220176e5f3ae8a4638a7731e37de762 Mon Sep 17 00:00:00 2001 From: Dr-Daily <56745553+Dr-Daily@users.noreply.github.com> Date: Sat, 20 Mar 2021 10:26:46 -0600 Subject: [PATCH 1/9] Added Hacks for the 2020 version --- .gitignore | 1 + create_j1939db-json.py | 39 +++++++++++++++++++++++++++------------ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 6499210..fbca0fa 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ MANIFEST # Excel spreadsheets *.xls *.xlsx +j1939db.json diff --git a/create_j1939db-json.py b/create_j1939db-json.py index 60e371e..5e1e0e6 100644 --- a/create_j1939db-json.py +++ b/create_j1939db-json.py @@ -119,7 +119,9 @@ def get_spn_resolution(contents): left = J1939daConverter.just_numerals(left.split(' ')[0]) right = J1939daConverter.just_numerals(right.split(' ')[0]) return asteval.Interpreter()('%s/%s' % (left, right)) - elif 'microsiemens/mm' in norm_contents or 'usiemens/mm' in norm_contents: # special handling for weirdness + elif 'microsiemens/mm' in norm_contents or \ + 'usiemens/mm' in norm_contents or \ + 'kw/s' in norm_contents: # special handling for this weirdness return float(contents.split(' ')[0]) raise ValueError('unknown spn resolution "%s"' % contents) @@ -280,8 +282,16 @@ def create_bit_object_from_description(spn_description, bit_object): first_val = int(range_boundaries[0], base=16) second_val = int(range_boundaries[1], base=16) else: - first_val = int(range_boundaries[0], base=10) - second_val = int(range_boundaries[1], base=10) + try: + first_val = int(range_boundaries[0], base=10) + second_val = int(range_boundaries[1], base=10) + except ValueError: + #Try binary + first = re.sub(r'b', '', range_boundaries[0]) + first_val = (int(first, base=2)) + second = re.sub(r'b', '', range_boundaries[1]) + second_val = (int(second, base=2)) + for i in range(first_val, second_val+1): bit_object.update(({str(i): enum_description})) @@ -294,7 +304,12 @@ def create_bit_object_from_description(spn_description, bit_object): elif 'x' in first.lower(): val = str(int(first, base=16)) else: - val = str(int(first, base=10)) + try: + val = str(int(first, base=10)) + except ValueError: + #Try binary + first = re.sub(r'b', '', first) + val = str(int(first, base=2)) bit_object.update(({val: enum_description})) @@ -314,22 +329,22 @@ def process_spns_and_pgns_tab(self, sheet): spn_factcheck_map = dict() header_row, header_row_num = self.get_header_row(sheet) - + #print(header_row) pgn_col = header_row.index('PGN') spn_col = header_row.index('SPN') - acronym_col = header_row.index('ACRONYM') - pgn_label_col = header_row.index('PARAMETER_GROUP_LABEL') - pgn_data_length_col = header_row.index('PGN_DATA_LENGTH') + acronym_col = header_row.index('PG_ACRONYM') + pgn_label_col = header_row.index('PG_LABEL') + pgn_data_length_col = header_row.index('PG_DATA_LENGTH') transmission_rate_col = header_row.index('TRANSMISSION_RATE') - spn_position_in_pgn_col = header_row.index('SPN_POSITION_IN_PGN') - spn_name_col = header_row.index('SPN_NAME') + spn_position_in_pgn_col = header_row.index('SP_POSITION_IN_PG') + spn_name_col = header_row.index('SP_LABEL') offset_col = header_row.index('OFFSET') data_range_col = header_row.index('DATA_RANGE') resolution_col = header_row.index('RESOLUTION') - spn_length_col = header_row.index('SPN_LENGTH') + spn_length_col = header_row.index('SP_LENGTH') units_col = header_row.index('UNITS') operational_range_col = header_row.index('OPERATIONAL_RANGE') - spn_description_col = header_row.index('SPN_DESCRIPTION') + spn_description_col = header_row.index('SP_DESCRIPTION') for i in range(header_row_num+1, sheet.nrows): row = sheet.row_values(i) From 01438e57ab0eb89139213a0c1097c0b88e9f3d2e Mon Sep 17 00:00:00 2001 From: Dr-Daily <56745553+Dr-Daily@users.noreply.github.com> Date: Thu, 25 Mar 2021 14:42:37 -0600 Subject: [PATCH 2/9] Removed incomplete file --- J1939db.json | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 J1939db.json diff --git a/J1939db.json b/J1939db.json deleted file mode 100644 index 58c216e..0000000 --- a/J1939db.json +++ /dev/null @@ -1,11 +0,0 @@ -{ -"COMMENT": "These are not the DBs you are looking for. See HOWTO in README.md", -"J1939PGNdb": { -}, -"J1939SPNdb": { -}, -"J1939BitDecodings": { -}, -"J1939SATabledb": { -} -} \ No newline at end of file From bc4d5f2a744fa451f9fd749592df836e5955ff08 Mon Sep 17 00:00:00 2001 From: Ben Gardiner Date: Tue, 25 Jan 2022 13:12:29 -0500 Subject: [PATCH 3/9] don't ignore and delete the default (currently empty) J1939db.json --- .gitignore | 1 - J1939db.json | 11 +++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 J1939db.json diff --git a/.gitignore b/.gitignore index fbca0fa..6499210 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,3 @@ MANIFEST # Excel spreadsheets *.xls *.xlsx -j1939db.json diff --git a/J1939db.json b/J1939db.json new file mode 100644 index 0000000..58c216e --- /dev/null +++ b/J1939db.json @@ -0,0 +1,11 @@ +{ +"COMMENT": "These are not the DBs you are looking for. See HOWTO in README.md", +"J1939PGNdb": { +}, +"J1939SPNdb": { +}, +"J1939BitDecodings": { +}, +"J1939SATabledb": { +} +} \ No newline at end of file From 8dfcd43897e7658e79ba4e482157e2430a35a766 Mon Sep 17 00:00:00 2001 From: Ben Gardiner Date: Tue, 25 Jan 2022 13:14:50 -0500 Subject: [PATCH 4/9] improve enum line identifying in matching but also include some blocklists necessary to avoid incorrectly identifying them as enum lines in newer DAs --- create_j1939db-json.py | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/create_j1939db-json.py b/create_j1939db-json.py index 5e1e0e6..3aea6de 100644 --- a/create_j1939db-json.py +++ b/create_j1939db-json.py @@ -210,23 +210,41 @@ def get_spn_start_bit(contents): return pos_pair + @staticmethod + def is_enum_line(line): + if line.lower().startswith('bit state'): + return True + elif re.match(r'^[ ]*[0-9][0-9bxXA-F\-:]*[ ]+[^ ]+', line): + return True + return False + @staticmethod def get_enum_lines(description_lines): enum_lines = list() + + def add_enum_line(test_line): + test_line = re.sub(r'(Bit States|Bit State)', '', test_line, flags=re.IGNORECASE) + if any(e in test_line for e in [': Tokyo', ' SPN 8846 ', ' SPN 8842 ', ' SPN 3265 ', ' SPN 3216 ', '13 preprogrammed intermediate ', '3 ASCII space characters']): + return False + enum_lines.append(test_line) + return True + + any_found = False for line in description_lines: - if line.lower().startswith('bit state'): - line = re.sub(r'(Bit States|Bit State)', '', line, flags=re.IGNORECASE) - enum_lines.append(line) - elif re.match(r'^[ ]*[0-9][0-9bxXA-F\-:]*[ ]+[^ ]+', line): - enum_lines.append(line) + if J1939daConverter.is_enum_line(line): + if any_found: + add_enum_line(line) + else: + if J1939daConverter.match_single_enum_line(line): # special handling: first enum must use single assignment + any_found = add_enum_line(line) return enum_lines @staticmethod - def is_enum_lines_binary(description_lines): + def is_enum_lines_binary(enum_lines_only): all_ones_and_zeroes = True - for line in description_lines: - first = re.match(ENUM_SINGLE_LINE_RE, line).groups()[0] + for line in enum_lines_only: + first = J1939daConverter.match_single_enum_line(line).groups()[0] if re.sub(r'[^10b]', '', first) != first: all_ones_and_zeroes = False break @@ -245,6 +263,12 @@ def get_enum_line_range(line): else: return None + @staticmethod + def match_single_enum_line(line): + line = re.sub(r'[ ]+', ' ', line) + line = re.sub(r'[ ]?\-\-[ ]?', ' = ', line) + return re.match(ENUM_SINGLE_LINE_RE, line) + @staticmethod # returns the description part (just that part) of an enum line def get_enum_line_description(line): @@ -254,7 +278,7 @@ def get_enum_line_description(line): if match: line = match.groups()[-1] else: - match = re.match(ENUM_SINGLE_LINE_RE, line) + match = J1939daConverter.match_single_enum_line(line) if match: line = match.groups()[-1] line = line.strip() From 05b3c9ee5e5abdd0be5e45552b4c06e07efab20d Mon Sep 17 00:00:00 2001 From: Ben Gardiner Date: Tue, 25 Jan 2022 13:15:46 -0500 Subject: [PATCH 5/9] with the update enum line identification the try/except blocks for tryirying last-ditch effort as binary aren't necessary anymore --- create_j1939db-json.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/create_j1939db-json.py b/create_j1939db-json.py index 3aea6de..4465a18 100644 --- a/create_j1939db-json.py +++ b/create_j1939db-json.py @@ -306,16 +306,8 @@ def create_bit_object_from_description(spn_description, bit_object): first_val = int(range_boundaries[0], base=16) second_val = int(range_boundaries[1], base=16) else: - try: - first_val = int(range_boundaries[0], base=10) - second_val = int(range_boundaries[1], base=10) - except ValueError: - #Try binary - first = re.sub(r'b', '', range_boundaries[0]) - first_val = (int(first, base=2)) - second = re.sub(r'b', '', range_boundaries[1]) - second_val = (int(second, base=2)) - + first_val = int(range_boundaries[0], base=10) + second_val = int(range_boundaries[1], base=10) for i in range(first_val, second_val+1): bit_object.update(({str(i): enum_description})) @@ -328,12 +320,7 @@ def create_bit_object_from_description(spn_description, bit_object): elif 'x' in first.lower(): val = str(int(first, base=16)) else: - try: - val = str(int(first, base=10)) - except ValueError: - #Try binary - first = re.sub(r'b', '', first) - val = str(int(first, base=2)) + val = str(int(first, base=10)) bit_object.update(({val: enum_description})) From e25f331e0ddf32077dafe732ccdfd32b3bfc239a Mon Sep 17 00:00:00 2001 From: Ben Gardiner Date: Tue, 25 Jan 2022 13:16:30 -0500 Subject: [PATCH 6/9] also add a blocklist entry for 'data specific' when looking at 'resolution' / 'scaling' in new DAs --- create_j1939db-json.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/create_j1939db-json.py b/create_j1939db-json.py index 4465a18..a3af6f6 100644 --- a/create_j1939db-json.py +++ b/create_j1939db-json.py @@ -99,7 +99,9 @@ def just_numerals(contents): # returns a float in X per bit or int(0) def get_spn_resolution(contents): norm_contents = contents.lower() - if '0 to 255 per byte' in norm_contents or 'states/' in norm_contents: + if '0 to 255 per byte' in norm_contents or \ + ' states' in norm_contents or \ + norm_contents == 'data specific': return 1.0 elif 'bit-mapped' in norm_contents or \ 'binary' in norm_contents or \ From 516f9d10e8f9abf068d60d443ff1d0fddbef6fbd Mon Sep 17 00:00:00 2001 From: Ben Gardiner Date: Tue, 25 Jan 2022 13:17:05 -0500 Subject: [PATCH 7/9] add synonym headers and grab the first match to handle old and new DAs --- create_j1939db-json.py | 54 ++++++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/create_j1939db-json.py b/create_j1939db-json.py index a3af6f6..a464ad7 100644 --- a/create_j1939db-json.py +++ b/create_j1939db-json.py @@ -342,22 +342,30 @@ def process_spns_and_pgns_tab(self, sheet): spn_factcheck_map = dict() header_row, header_row_num = self.get_header_row(sheet) - #print(header_row) - pgn_col = header_row.index('PGN') - spn_col = header_row.index('SPN') - acronym_col = header_row.index('PG_ACRONYM') - pgn_label_col = header_row.index('PG_LABEL') - pgn_data_length_col = header_row.index('PG_DATA_LENGTH') - transmission_rate_col = header_row.index('TRANSMISSION_RATE') - spn_position_in_pgn_col = header_row.index('SP_POSITION_IN_PG') - spn_name_col = header_row.index('SP_LABEL') - offset_col = header_row.index('OFFSET') - data_range_col = header_row.index('DATA_RANGE') - resolution_col = header_row.index('RESOLUTION') - spn_length_col = header_row.index('SP_LENGTH') - units_col = header_row.index('UNITS') - operational_range_col = header_row.index('OPERATIONAL_RANGE') - spn_description_col = header_row.index('SP_DESCRIPTION') + pgn_col = self.get_any_header_column(header_row, 'PGN') + spn_col = self.get_any_header_column(header_row, 'SPN') + acronym_col = self.get_any_header_column(header_row, + ['ACRONYM', 'PG_ACRONYM']) + pgn_label_col = self.get_any_header_column(header_row, + ['PARAMETER_GROUP_LABEL', 'PG_LABEL']) + pgn_data_length_col = self.get_any_header_column(header_row, + ['PGN_DATA_LENGTH', 'PG_DATA_LENGTH']) + transmission_rate_col = self.get_any_header_column(header_row, 'TRANSMISSION_RATE') + spn_position_in_pgn_col = self.get_any_header_column(header_row, + ['SPN_POSITION_IN_PGN','SP_POSITION_IN_PG']) + spn_name_col = self.get_any_header_column(header_row, + ['SPN_NAME', 'SP_LABEL']) + offset_col = self.get_any_header_column(header_row, 'OFFSET') + data_range_col = self.get_any_header_column(header_row, 'DATA_RANGE') + resolution_col = self.get_any_header_column(header_row, + ['RESOLUTION', 'SCALING']) + spn_length_col = self.get_any_header_column(header_row, + ['SPN_LENGTH', 'SP_LENGTH']) + units_col = self.get_any_header_column(header_row, + ['UNITS', 'UNIT']) + operational_range_col = self.get_any_header_column(header_row, 'OPERATIONAL_RANGE') + spn_description_col = self.get_any_header_column(header_row, + ['SPN_DESCRIPTION', 'SP_DESCRIPTION']) for i in range(header_row_num+1, sheet.nrows): row = sheet.row_values(i) @@ -506,6 +514,16 @@ def process_spns_and_pgns_tab(self, sheet): return + def get_any_header_column(self, header_row, header_texts): + if not isinstance(header_texts, list): + header_texts = [header_texts] + for t in header_texts: + try: + return header_row.index(t) + except ValueError: + continue + return -1 + def get_header_row(self, sheet): header_row_num = self.lookup_header_row(sheet) @@ -649,8 +667,8 @@ def process_any_source_addresses_sheet(self, sheet): header_row, header_row_num = self.get_header_row(sheet) - source_address_id_col = header_row.index('SOURCE_ADDRESS_ID') - name_col = header_row.index('NAME') + source_address_id_col = self.get_any_header_column(header_row, 'SOURCE_ADDRESS_ID') + name_col = self.get_any_header_column(header_row, 'NAME') for i in range(header_row_num+1, sheet.nrows): row = sheet.row_values(i) From 863af661d410e4875c71c7ba4f951eb88cd3efe2 Mon Sep 17 00:00:00 2001 From: Ben Gardiner Date: Tue, 25 Jan 2022 13:17:55 -0500 Subject: [PATCH 8/9] add synonym sheets and grab the first match to handle old and new DAs --- create_j1939db-json.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/create_j1939db-json.py b/create_j1939db-json.py index a464ad7..8c77762 100644 --- a/create_j1939db-json.py +++ b/create_j1939db-json.py @@ -685,7 +685,7 @@ def process_any_source_addresses_sheet(self, sheet): def convert(self, output_file): self.j1939db = OrderedDict() - sheet_name = 'SPNs & PGNs' + sheet_name = ['SPNs & PGNs', 'SPs & PGs'] self.process_spns_and_pgns_tab(self.find_first_sheet_by_name(sheet_name)) sheet_name = 'Global Source Addresses (B2)' self.process_any_source_addresses_sheet(self.find_first_sheet_by_name(sheet_name)) @@ -704,11 +704,14 @@ def convert(self, output_file): return - def find_first_sheet_by_name(self, sheet_name): - for book in self.digital_annex_xls_list: - if sheet_name in book.sheet_names(): - sheet = book.sheet_by_name(sheet_name) - return sheet + def find_first_sheet_by_name(self, sheet_names): + if not isinstance(sheet_names, list): + sheet_names = [sheet_names] + for sheet_name in sheet_names: + for book in self.digital_annex_xls_list: + if sheet_name in book.sheet_names(): + sheet = book.sheet_by_name(sheet_name) + return sheet return None From f5b8585774d7270828eb099af50f6ce3a309e6c0 Mon Sep 17 00:00:00 2001 From: Ben Gardiner Date: Tue, 25 Jan 2022 13:20:19 -0500 Subject: [PATCH 9/9] xlsx files are not supported --- .gitignore | 1 - create_j1939db-json.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 6499210..7e76621 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,3 @@ MANIFEST # Excel spreadsheets *.xls -*.xlsx diff --git a/create_j1939db-json.py b/create_j1939db-json.py index 8c77762..e574117 100644 --- a/create_j1939db-json.py +++ b/create_j1939db-json.py @@ -24,7 +24,7 @@ parser = argparse.ArgumentParser() parser.add_argument('-f', '--digital_annex_xls', type=str, required=True, action='append', default=[], nargs='+', - help="the J1939 Digital Annex excel sheet used as input") + help="the J1939 Digital Annex .xls excel file used as input") parser.add_argument('-w', '--write-json', type=str, default='-', help="where to write the output. defaults to stdout") args = parser.parse_args()