Issue 483 encoding errors

ebu_tt_live/bindings/__init__.py

@@ -939,7 +939,7 @@ def _merge_deconflict_ids(cls, element, dest, ids):
         output = []
 
         for item in children:
-            log.debug('processing child: {} of {}'.format(item.value, element))
+            #log.debug('processing child: {} of {}'.format(item.value, element))
             if isinstance(item, NonElementContent):
                 copied_stuff = copy.copy(item.value)
                 output.append(copied_stuff)

ebu_tt_live/carriage/filesystem.py

@@ -9,6 +9,7 @@
 import six
 import os
 import time
+import codecs
 
 
 log = logging.getLogger(__name__)
@@ -72,20 +73,21 @@ def __init__(self,
                  file_name_pattern = CFG_FILENAME_PATTERN, 
                  message_file_name_pattern = CFG_MESSAGE_PATTERN, 
                  circular_buf_size = 0, 
-                 suppress_manifest = False):
+                 suppress_manifest = False,
+                 first_msg_counter = 0):
         self._dirpath = dirpath
         if not os.path.exists(self._dirpath):
             os.makedirs(self._dirpath)
         self._file_name_pattern = file_name_pattern
         self._message_file_name_pattern = message_file_name_pattern
         self._counter = 0
+        self._msg_counter = first_msg_counter
         self._circular_buf_size = circular_buf_size
         if circular_buf_size > 0 :
             self._circular_buf = RotatingFileBuffer(maxlen=circular_buf_size)
         self._suppress_manifest = suppress_manifest
         # Get a set of default clocks
         self._default_clocks = {}
-        self._msg_counter = 0
 
     def _get_default_clock(self, sequence_identifier, time_base, clock_mode=None):
         clock_obj = self._default_clocks.get(sequence_identifier, None)
@@ -153,7 +155,7 @@ def emit_data(self, data, sequence_identifier=None, sequence_number=None,
         # can be selected once at the beginning and dereferenced rather than repeating
         # if statements.
         filepath = os.path.join(self._dirpath, filename)
-        with open(filepath, 'w') as destfile:
+        with codecs.open(filepath, mode='w', errors='ignore') as destfile:
             destfile.write(data)
             destfile.flush()
 
@@ -198,7 +200,7 @@ def emit_data(self, data, sequence_identifier=None, sequence_number=None,
             new_manifest_line = CFG_MANIFEST_LINE_PATTERN.format(
                 availability_time=timedelta_to_str_manifest(availability_time), 
                 filename=filename)
-            with open(self._manifest_path, 'a') as f:
+            with codecs.open(self._manifest_path, mode='a', errors='ignore') as f:
                 f.write(new_manifest_line)
 
 
@@ -236,11 +238,11 @@ def __init__(self, manifest_path, custom_consumer, do_tail):
         self._manifest_path = manifest_path
         self._custom_consumer = custom_consumer
         self._do_tail = do_tail
-        with open(manifest_path, 'r') as manifest:
+        with codecs.open(manifest_path, 'r') as manifest:
             self._manifest_lines_iter = iter(manifest.readlines())
 
     def resume_reading(self):
-        with open(self._manifest_path, 'r') as manifest_file:
+        with codecs.open(self._manifest_path, 'r') as manifest_file:
             while True:
                 manifest_line = manifest_file.readline()
                 if not manifest_line:
@@ -256,7 +258,7 @@ def resume_reading(self):
                     availability_time_str, xml_file_name = manifest_line.rstrip().split(',')
                     xml_file_path = os.path.join(self._dirpath, xml_file_name)
                     xml_content = None
-                    with open(xml_file_path, 'r') as xml_file:
+                    with codecs.open(xml_file_path, 'r') as xml_file:
                         xml_content = xml_file.read()
                     data = [availability_time_str, xml_content]
                     self._custom_consumer.on_new_data(data)

ebu_tt_live/config/backend.py

@@ -137,8 +137,6 @@ def _ws_create_server_factory(self, listen, producer=None, consumer=None):
 
     def _ws_create_client_factories(self, connect, producer=None, consumer=None, proxy=None):
         factory_args = {}
-        if proxy:
-            factory_args.update({'host': proxy.host, 'port': proxy.port})
         for dst in connect:
             client_factory = self._websocket.BroadcastClientFactory(
                 url=dst.geturl(),
@@ -147,6 +145,8 @@ def _ws_create_client_factories(self, connect, producer=None, consumer=None, pro
                 **factory_args
             )
             client_factory.protocol = self._websocket.BroadcastClientProtocol
+            client_factory.proxy = proxy
+
             client_factory.connect()
 
     def ws_backend_producer(self, custom_producer, listen=None, connect=None, proxy=None):

ebu_tt_live/config/carriage.py

@@ -2,7 +2,6 @@
 from ebu_tt_live.carriage.direct import DirectCarriageImpl
 from ebu_tt_live.carriage.websocket import WebsocketProducerCarriage, WebsocketConsumerCarriage
 from ebu_tt_live.carriage import filesystem
-from ebu_tt_live.utils import HTTPProxyConfig
 from ebu_tt_live.strings import ERR_CONF_PROXY_CONF_VALUE, ERR_NO_SUCH_COMPONENT
 from ebu_tt_live.errors import ConfigurationError
 from ebu_tt_live.strings import CFG_FILENAME_PATTERN, CFG_MESSAGE_PATTERN
@@ -76,6 +75,10 @@ class FilesystemOutput(ConfigurableComponent):
         default=False,
         doc='Suppress output of a manifest file (default false)'
     )
+    required_config.add_option(
+        'begin_count',
+        default=0,
+        doc='Value to begin counting at for patterns including {counter}; the first output value will be this plus 1.')
 
     def __init__(self, config, local_config):
         super(FilesystemOutput, self).__init__(config, local_config)
@@ -84,7 +87,8 @@ def __init__(self, config, local_config):
             file_name_pattern=self.config.filename_pattern,
             message_file_name_pattern=self.config.message_filename_pattern,
             circular_buf_size=self.config.rotating_buf,
-            suppress_manifest=self.config.suppress_manifest)
+            suppress_manifest=self.config.suppress_manifest,
+            first_msg_counter=self.config.begin_count)
 
 
 
@@ -134,10 +138,7 @@ def parse_proxy_address(value):
     match = proxy_regex.match(value)
     if match:
         # Ignoring the protocol part for now as it is only a http proxy
-        result = HTTPProxyConfig(
-            host=match.group('host'),
-            port=int(match.group('port'))
-        )
+        result = {u'host': match.group('host'), u'port': int(match.group('port'))}
     elif value:
         # In this case something was provided that isn't a falsy value but the parsing failed.
         raise ConfigurationError(

ebu_tt_live/node/deduplicator.py

@@ -55,7 +55,7 @@ def remove_duplication(self, document):
 
         if document.binding.head.styling is not None:
             styles = document.binding.head.styling.style
-            print styles
+
             document.binding.head.styling.style = None
 
             self.CollateUniqueVals(styles, old_id_dict, new_id_dict, hash_dict)

ebu_tt_live/utils.py

@@ -358,8 +358,6 @@ def __call__(cls, *args, **kwargs):
         instance = super(AutoRegisteringABCMeta, cls).__call__(*args, **kwargs)
         return instance
 
-HTTPProxyConfig = collections.namedtuple('HTTPProxyConfig', ['host', 'port'])
-
 
 # The following section is taken from https://github.com/django/django/blob/master/django/test/utils.py
 # This is a relatively simple XML comparator implementation based on Python's minidom library.
@@ -467,4 +465,4 @@ def first_node(document):
     want_root = first_node(parseString(want))
     got_root = first_node(parseString(got))
 
-    return check_element(want_root, got_root)
+    return check_element(want_root, got_root)

requirements.txt

@@ -9,7 +9,6 @@ sphinx-rtd-theme
 pytest-bdd
 pytest-cov
 pytest-mock
-pytest-capturelog
 pytest-twisted
 coverage
 pytest-runner