Merge remote-tracking branch 'origin/python3'

docs/extras.rst

@@ -226,7 +226,7 @@ the list reported differences is not guaranteed to be complete. Conversely, if
 no differences are reported, then the YAML files are guaranteed to have the
 same content.
 
-``test.sh``
+``sync_test``
 -----------
 
 To keep the Python- and JavaScript implementations in sync, we use a shell
@@ -235,7 +235,7 @@ examples.
 
 ::
 
-    bash extras/test.sh
+    ./extras/sync_test
 
 This will perform a parser test and an invariance test for all examples.
 

docs/library.rst

@@ -64,7 +64,7 @@ Now we can initialise the parser using an instance of the new class:
 .. code:: python
 
     parser = bin_parser.BinReader(
-        open('something.dat').read(),
+        open('something.dat', 'rb').read(),
         yaml.safe_load(open('structure.yml')),
         yaml.safe_load(open('types.yml')),
         functions=Invert())

examples/balance/parser.py

@@ -5,10 +5,10 @@
 
 
 parser = BinReader(
-    open('balance.dat').read(),
+    open('balance.dat', 'rb').read(),
     yaml.safe_load(open('structure.yml')),
     yaml.safe_load(open('types.yml')))
 
-print parser.parsed['name']
-print parser.parsed['year_of_birth']
-print parser.parsed['balance']
+print('{}\n'.format(parser.parsed['name']))
+print('{}\n'.format(parser.parsed['year_of_birth']))
+print('{}\n'.format(parser.parsed['balance']))

examples/prince/python/reader.py

@@ -8,7 +8,7 @@
 
 
 parser = BinReader(
-    open('../prince.hof').read(),
+    open('../prince.hof', 'rb').read(),
     yaml.safe_load(open('../structure.yml')),
     yaml.safe_load(open('../types.yml')),
     functions=PrinceReadFunctions())

examples/prince/python/writer.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+import os
 import sys
 
 import yaml
@@ -12,4 +13,6 @@
     yaml.safe_load(open('../structure.yml')),
     yaml.safe_load(open('../types.yml')),
     functions=PrinceWriteFunctions())
-sys.stdout.write(parser.data)
+
+fp = os.fdopen(sys.stdout.fileno(), 'wb')
+fp.write(parser.data)

extras/compare_yaml.py

@@ -17,20 +17,20 @@ def dict_compare(d1, d2):
     :arg dict d2: An other nested dictionary.
     """
     if type(d1) == dict:
-        for key in set(d1.keys() + d2.keys()):
+        for key in set(list(d1.keys()) + list(d2.keys())):
             if key in d1 and key in d2:
                 dict_compare(d1[key], d2[key])
             else:
-                print 'missing key: {}'.format(key)
+                print('missing key: {}\n'.format(key))
     elif type(d1) == list:
         if len(d1) != len(d2):
-            print 'lists of different length'
+            print('lists of different length\n')
         else:
             for index in range(len(d1)):
                 dict_compare(d1[index], d2[index])
     else:
         if d1 != d2:
-            print '{} != {}'.format(d1, d2)
+            print('{} != {}\n'.format(d1, d2))
 
 
 def yaml_compare(input_handle_1, input_handle_2):

extras/make_skeleton.py

@@ -30,7 +30,8 @@ def make_structure(data, delimiter):
     """
     structure = []
 
-    for index in range(len(data.split(''.join(map(chr, delimiter))))):
+    for index in range(
+            len(data.split(''.join(map(chr, delimiter)).encode('utf-8')))):
         structure.append({
             'name': 'field_{:06d}'.format(index),
             'type': 'raw'})
@@ -42,11 +43,11 @@ def make_types(delimiter):
     return {
         'types': {
             'raw': {
-                'delimiter': map(HexInt, delimiter),
+                'delimiter': list(map(HexInt, delimiter)),
                 'function': {
                     'name': 'raw'}},
             'text': {
-                'delimiter': map(HexInt, delimiter)}}}
+                'delimiter': list(map(HexInt, delimiter))}}}
 
 
 def make_skeleton(input_handle, structure_handle, types_handle, delimiter):
@@ -78,7 +79,7 @@ def main():
         formatter_class=argparse.RawDescriptionHelpFormatter)
 
     parser.add_argument(
-        'input_handle', metavar='INPUT', type=argparse.FileType('r'),
+        'input_handle', metavar='INPUT', type=argparse.FileType('rb'),
         help='input file')
     parser.add_argument(
         'structure_handle', metavar='STRUCTURE', type=argparse.FileType('w'),

python/bin_parser.py

@@ -220,7 +220,7 @@ def _get_field(self, size=0, delimiter=[]):
         if self._offset >= len(self.data):
             raise StopIteration
 
-        separator = ''.join(chr(c) for c in delimiter)
+        separator = ''.join(chr(c) for c in delimiter).encode('utf-8')
 
         if size:
             # Fixed sized field.
@@ -323,7 +323,7 @@ def _parse_while(self, item, dest, name):
             dest[name].append({})
             self._parse([delim], dest[name][-1])
 
-        dest[item['while']['term']] = dest[name].pop(-1).values()[0]
+        dest[item['while']['term']] = list(dest[name].pop(-1).values())[0]
 
     def _parse(self, structure, dest):
         """Parse a binary file.
@@ -398,7 +398,7 @@ def __init__(
         super(BinWriter, self).__init__(
             structure, types, functions, debug, log)
 
-        self.data = ''
+        self.data = b''
         self.parsed = parsed
 
         self._encode(self._structure, self.parsed)
@@ -415,10 +415,10 @@ def _set_field(self, data, size=0, delimiter=[]):
 
         if delimiter:
             # Add the delimiter for variable length fields.
-            field += ''.join(chr(c) for c in delimiter)
+            field += ''.join(chr(c) for c in delimiter).encode('utf-8')
 
         # Pad the field if necessary.
-        field += chr(0x00) * (size - len(field))
+        field += b'\x00' * (size - len(field))
 
         if size:
             # Clip the field if it is too large.

python/cli.py

@@ -24,7 +24,7 @@ def bin_reader(
         yaml.safe_load(structure_handle),
         yaml.safe_load(types_handle),
         prune=prune, debug=debug)
-    output_handle.write(u'---\n')
+    output_handle.write('---\n')
     yaml.safe_dump(
         parser.parsed, output_handle, width=76, default_flow_style=False)
     if debug:
@@ -53,19 +53,32 @@ def bin_writer(
 
 def main():
     """Command line argument parsing."""
-    opt_parser = argparse.ArgumentParser(add_help=False)
-    opt_parser.add_argument(
+    bin_input_parser = argparse.ArgumentParser(add_help=False)
+    bin_input_parser.add_argument(
+        'input_handle', metavar='INPUT', type=argparse.FileType('rb'),
+        help='input file')
+
+    input_parser = argparse.ArgumentParser(add_help=False)
+    input_parser.add_argument(
         'input_handle', metavar='INPUT', type=argparse.FileType('r'),
         help='input file')
+
+    bin_output_parser = argparse.ArgumentParser(add_help=False)
+    bin_output_parser.add_argument(
+        'output_handle', metavar='OUTPUT', type=argparse.FileType('wb'),
+        help='output file')
+    output_parser = argparse.ArgumentParser(add_help=False)
+    output_parser.add_argument(
+        'output_handle', metavar='OUTPUT', type=argparse.FileType('w'),
+        help='output file')
+
+    opt_parser = argparse.ArgumentParser(add_help=False)
     opt_parser.add_argument(
         'structure_handle', metavar='STRUCTURE', type=argparse.FileType('r'),
         help='structure definition file')
     opt_parser.add_argument(
         'types_handle', metavar='TYPES', type=argparse.FileType('r'),
         help='type definition file')
-    opt_parser.add_argument(
-        'output_handle', metavar='OUTPUT', type=argparse.FileType('w'),
-        help='output file')
     opt_parser.add_argument(
         '-d', dest='debug', type=int, default=0,
         help='debugging level (%(type)s default=%(default)s)')
@@ -75,17 +88,20 @@ def main():
         formatter_class=argparse.RawDescriptionHelpFormatter)
     parser.add_argument(
         '-v', action='version', version=version(parser.prog))
-    subparsers = parser.add_subparsers()
+    subparsers = parser.add_subparsers(dest='subcommand')
+    subparsers.required = True
 
     read_parser = subparsers.add_parser(
-        'read', parents=[opt_parser], description=doc_split(bin_reader))
+        'read', parents=[bin_input_parser, opt_parser, output_parser],
+        description=doc_split(bin_reader))
     read_parser.add_argument(
         '-p', dest='prune', default=False, action='store_true',
         help='remove unknown data fields')
     read_parser.set_defaults(func=bin_reader)
 
     write_parser = subparsers.add_parser(
-        'write', parents=[opt_parser], description=doc_split(bin_writer))
+        'write', parents=[input_parser, opt_parser, bin_output_parser],
+        description=doc_split(bin_writer))
     write_parser.set_defaults(func=bin_writer)
 
     try:

python/functions.py

@@ -1,4 +1,5 @@
 """Field packing and unpacking functions for the general binary parser."""
+import codecs
 import collections
 import operator
 import struct
@@ -57,6 +58,8 @@ def struct(self, data, fmt='b', labels=None, annotation=None):
             if labels:
                 return dict(zip(labels, decoded))
             return list(decoded)
+        if fmt == 'c':
+            return decoded[0].decode('utf-8')
         return decoded[0]
 
     def raw(self, data):
@@ -66,10 +69,11 @@ def raw(self, data):
 
         :returns str: Hexadecimal representation of {data}.
         """
-        raw_data = data.encode('hex')
+        raw_data = codecs.encode(data, 'hex')
 
-        return ' '.join(
-            [raw_data[x:x + 2] for x in range(0, len(raw_data), 2)])
+        return b' '.join(
+            [raw_data[x:x + 2] for x in range(0, len(raw_data), 2)]).decode(
+                'utf-8')
 
     def bit(self, data):
         return '{:08b}'.format(ord(data))
@@ -193,6 +197,8 @@ def struct(self, data, fmt='b', labels=None, annotation=None):
             data_list = [data[x] for x in labels]
         elif isinstance(data, list):
             data_list = data
+        elif fmt == 'c':
+            data_list = [data.encode('utf-8')]
         else:
             data_list = [data]
 
@@ -206,10 +212,10 @@ def struct(self, data, fmt='b', labels=None, annotation=None):
         return struct.pack(fmt, *data_list)
 
     def raw(self, hex_string):
-        return ''.join(hex_string.split()).decode('hex')
+        return codecs.decode(''.join(hex_string.split()), 'hex')
 
     def bit(self, bit_string):
-        return chr(int('0b{}'.format(bit_string), 2))
+        return chr(int('0b{}'.format(bit_string), 2)).encode('utf-8')
 
     def int(self, integer):
         # TODO: Deprecated, remove.
@@ -271,4 +277,4 @@ def flags(self, flags_dict, annotation):
                 else:
                     values += int(key[prefix_len:], 0x10)
 
-        return chr(values)
+        return chr(values).encode('utf-8')

tests/test_functions.py

@@ -22,70 +22,74 @@ def _idem(self, func, data, **kwargs):
             getattr(self._brf, func)(data, **kwargs), **kwargs) == data
 
     def test_raw_single(self):
-        assert self._brf.raw('\x03') == '03'
+        assert self._brf.raw(b'\x03') == '03'
 
     def test_raw_multi(self):
-        assert self._brf.raw('\x00\x01\x02') == '00 01 02'
+        assert self._brf.raw(b'\x00\x01\x02') == '00 01 02'
 
     def test_raw_idem(self):
-        self._idem('raw', '\x00\x01\x02')
+        self._idem('raw', b'\x00\x01\x02')
 
     def test_bit(self):
-        assert self._brf.bit('\x03') == '00000011'
+        assert self._brf.bit(b'\x03') == '00000011'
 
     def test_bit_idem(self):
-        self._idem('bit', '\x03')
+        self._idem('bit', b'\x03')
 
     def test_le_s_short(self):
-        assert self._brf.struct('\x01\x00', fmt='<h') == 1
+        assert self._brf.struct(b'\x01\x00', fmt='<h') == 1
 
     def test_be_s_short(self):
-        assert self._brf.struct('\x00\x01', fmt='>h') == 1
+        assert self._brf.struct(b'\x00\x01', fmt='>h') == 1
 
     def test_le_s_short_idem(self):
-        self._idem('struct', '\x01\x02', fmt='<h')
+        self._idem('struct', b'\x01\x02', fmt='<h')
 
     def test_be_s_float_idem(self):
-        self._idem('struct', '\x01\x02\x03\x04', fmt='>f')
+        self._idem('struct', b'\x01\x02\x03\x04', fmt='>f')
 
     def test_labels(self):
         assert self._brf.struct(
-            '\x01\x02', fmt='BB', labels=['a', 'b']) == {'a': 1, 'b': 2}
+            b'\x01\x02', fmt='BB', labels=['a', 'b']) == {'a': 1, 'b': 2}
 
     def test_labels_idem(self):
-        self._idem('struct', '\x01\x02', fmt='BB', labels=['a', 'b'])
+        self._idem('struct', b'\x01\x02', fmt='BB', labels=['a', 'b'])
 
     def test_annotation(self):
         assert self._brf.struct(
-            '\x01\x02', fmt='BB', annotation={1: 'x'}) == ['x', 2]
+            b'\x01\x02', fmt='BB', annotation={1: 'x'}) == ['x', 2]
 
     def test_annotation_idem(self):
-        self._idem('struct', '\x01\x02', fmt='BB', annotation={1: 'a'})
+        self._idem('struct', b'\x01\x02', fmt='BB', annotation={1: 'a'})
 
     def test_labels_annotation(self):
         assert self._brf.struct(
-            '\x01\x02', fmt='BB', labels=['a', 'b'],
+            b'\x01\x02', fmt='BB', labels=['a', 'b'],
             annotation={1: 'x'}) == {'a': 'x', 'b': 2}
 
     def test_labels_annotation_idem(self):
         self._idem(
-            'struct', '\x01\x02', fmt='BB', labels=['a', 'b'],
+            'struct', b'\x01\x02', fmt='BB', labels=['a', 'b'],
             annotation={1: 'x'})
 
     def test_flags(self):
-        assert self._brf.flags('\x03', {}) == {'flag_01': True, 'flag_02': True}
+        assert self._brf.flags(b'\x03', {}) == {
+            'flag_01': True, 'flag_02': True}
 
     def test_flags_false(self):
-        assert self._bwf.flags({'flag_01': True, 'flag_02': False}, {}) == '\x01'
+        assert self._bwf.flags(
+            {'flag_01': True, 'flag_02': False}, {}) == b'\x01'
 
     def test_flags_idem(self):
-        self._idem('flags', '\x03', annotation={})
+        self._idem('flags', b'\x03', annotation={})
 
     def test_flags_annotation(self):
-        assert self._brf.flags('\x03', {2: 'a'}) == {'flag_01': True, 'a': True}
+        assert self._brf.flags(b'\x03', {2: 'a'}) == {
+            'flag_01': True, 'a': True}
 
     def test_flags_annotation_false(self):
-        assert self._brf.flags('\x01', {2: 'a'}) == {'flag_01': True, 'a': False}
+        assert self._brf.flags(b'\x01', {2: 'a'}) == {
+            'flag_01': True, 'a': False}
 
     def test_flags_annotation_idem(self):
-        self._idem('flags', '\x03', annotation={2: 'a'})
+        self._idem('flags', b'\x03', annotation={2: 'a'})