diff --git a/ebu_tt_live/bindings/__init__.py b/ebu_tt_live/bindings/__init__.py index 4b1eb4347..1381af0ee 100644 --- a/ebu_tt_live/bindings/__init__.py +++ b/ebu_tt_live/bindings/__init__.py @@ -939,7 +939,7 @@ def _merge_deconflict_ids(cls, element, dest, ids): output = [] for item in children: - log.debug('processing child: {} of {}'.format(item.value, element)) + #log.debug('processing child: {} of {}'.format(item.value, element)) if isinstance(item, NonElementContent): copied_stuff = copy.copy(item.value) output.append(copied_stuff) diff --git a/ebu_tt_live/carriage/filesystem.py b/ebu_tt_live/carriage/filesystem.py index ef4db4475..1b2434570 100644 --- a/ebu_tt_live/carriage/filesystem.py +++ b/ebu_tt_live/carriage/filesystem.py @@ -9,6 +9,7 @@ import six import os import time +import codecs log = logging.getLogger(__name__) @@ -72,20 +73,21 @@ def __init__(self, file_name_pattern = CFG_FILENAME_PATTERN, message_file_name_pattern = CFG_MESSAGE_PATTERN, circular_buf_size = 0, - suppress_manifest = False): + suppress_manifest = False, + first_msg_counter = 0): self._dirpath = dirpath if not os.path.exists(self._dirpath): os.makedirs(self._dirpath) self._file_name_pattern = file_name_pattern self._message_file_name_pattern = message_file_name_pattern self._counter = 0 + self._msg_counter = first_msg_counter self._circular_buf_size = circular_buf_size if circular_buf_size > 0 : self._circular_buf = RotatingFileBuffer(maxlen=circular_buf_size) self._suppress_manifest = suppress_manifest # Get a set of default clocks self._default_clocks = {} - self._msg_counter = 0 def _get_default_clock(self, sequence_identifier, time_base, clock_mode=None): clock_obj = self._default_clocks.get(sequence_identifier, None) @@ -153,7 +155,7 @@ def emit_data(self, data, sequence_identifier=None, sequence_number=None, # can be selected once at the beginning and dereferenced rather than repeating # if statements. filepath = os.path.join(self._dirpath, filename) - with open(filepath, 'w') as destfile: + with codecs.open(filepath, mode='w', errors='ignore') as destfile: destfile.write(data) destfile.flush() @@ -198,7 +200,7 @@ def emit_data(self, data, sequence_identifier=None, sequence_number=None, new_manifest_line = CFG_MANIFEST_LINE_PATTERN.format( availability_time=timedelta_to_str_manifest(availability_time), filename=filename) - with open(self._manifest_path, 'a') as f: + with codecs.open(self._manifest_path, mode='a', errors='ignore') as f: f.write(new_manifest_line) @@ -236,11 +238,11 @@ def __init__(self, manifest_path, custom_consumer, do_tail): self._manifest_path = manifest_path self._custom_consumer = custom_consumer self._do_tail = do_tail - with open(manifest_path, 'r') as manifest: + with codecs.open(manifest_path, 'r') as manifest: self._manifest_lines_iter = iter(manifest.readlines()) def resume_reading(self): - with open(self._manifest_path, 'r') as manifest_file: + with codecs.open(self._manifest_path, 'r') as manifest_file: while True: manifest_line = manifest_file.readline() if not manifest_line: @@ -256,7 +258,7 @@ def resume_reading(self): availability_time_str, xml_file_name = manifest_line.rstrip().split(',') xml_file_path = os.path.join(self._dirpath, xml_file_name) xml_content = None - with open(xml_file_path, 'r') as xml_file: + with codecs.open(xml_file_path, 'r') as xml_file: xml_content = xml_file.read() data = [availability_time_str, xml_content] self._custom_consumer.on_new_data(data) diff --git a/ebu_tt_live/config/backend.py b/ebu_tt_live/config/backend.py index c8c03b185..465c459e8 100644 --- a/ebu_tt_live/config/backend.py +++ b/ebu_tt_live/config/backend.py @@ -137,8 +137,6 @@ def _ws_create_server_factory(self, listen, producer=None, consumer=None): def _ws_create_client_factories(self, connect, producer=None, consumer=None, proxy=None): factory_args = {} - if proxy: - factory_args.update({'host': proxy.host, 'port': proxy.port}) for dst in connect: client_factory = self._websocket.BroadcastClientFactory( url=dst.geturl(), @@ -147,6 +145,8 @@ def _ws_create_client_factories(self, connect, producer=None, consumer=None, pro **factory_args ) client_factory.protocol = self._websocket.BroadcastClientProtocol + client_factory.proxy = proxy + client_factory.connect() def ws_backend_producer(self, custom_producer, listen=None, connect=None, proxy=None): diff --git a/ebu_tt_live/config/carriage.py b/ebu_tt_live/config/carriage.py index b1f0f1624..a8aa401d6 100644 --- a/ebu_tt_live/config/carriage.py +++ b/ebu_tt_live/config/carriage.py @@ -2,7 +2,6 @@ from ebu_tt_live.carriage.direct import DirectCarriageImpl from ebu_tt_live.carriage.websocket import WebsocketProducerCarriage, WebsocketConsumerCarriage from ebu_tt_live.carriage import filesystem -from ebu_tt_live.utils import HTTPProxyConfig from ebu_tt_live.strings import ERR_CONF_PROXY_CONF_VALUE, ERR_NO_SUCH_COMPONENT from ebu_tt_live.errors import ConfigurationError from ebu_tt_live.strings import CFG_FILENAME_PATTERN, CFG_MESSAGE_PATTERN @@ -76,6 +75,10 @@ class FilesystemOutput(ConfigurableComponent): default=False, doc='Suppress output of a manifest file (default false)' ) + required_config.add_option( + 'begin_count', + default=0, + doc='Value to begin counting at for patterns including {counter}; the first output value will be this plus 1.') def __init__(self, config, local_config): super(FilesystemOutput, self).__init__(config, local_config) @@ -84,7 +87,8 @@ def __init__(self, config, local_config): file_name_pattern=self.config.filename_pattern, message_file_name_pattern=self.config.message_filename_pattern, circular_buf_size=self.config.rotating_buf, - suppress_manifest=self.config.suppress_manifest) + suppress_manifest=self.config.suppress_manifest, + first_msg_counter=self.config.begin_count) @@ -134,10 +138,7 @@ def parse_proxy_address(value): match = proxy_regex.match(value) if match: # Ignoring the protocol part for now as it is only a http proxy - result = HTTPProxyConfig( - host=match.group('host'), - port=int(match.group('port')) - ) + result = {u'host': match.group('host'), u'port': int(match.group('port'))} elif value: # In this case something was provided that isn't a falsy value but the parsing failed. raise ConfigurationError( diff --git a/ebu_tt_live/node/deduplicator.py b/ebu_tt_live/node/deduplicator.py index e58364aeb..ced39e88d 100644 --- a/ebu_tt_live/node/deduplicator.py +++ b/ebu_tt_live/node/deduplicator.py @@ -55,7 +55,7 @@ def remove_duplication(self, document): if document.binding.head.styling is not None: styles = document.binding.head.styling.style - print styles + document.binding.head.styling.style = None self.CollateUniqueVals(styles, old_id_dict, new_id_dict, hash_dict) diff --git a/ebu_tt_live/utils.py b/ebu_tt_live/utils.py index 000598249..9ae3afca2 100644 --- a/ebu_tt_live/utils.py +++ b/ebu_tt_live/utils.py @@ -358,8 +358,6 @@ def __call__(cls, *args, **kwargs): instance = super(AutoRegisteringABCMeta, cls).__call__(*args, **kwargs) return instance -HTTPProxyConfig = collections.namedtuple('HTTPProxyConfig', ['host', 'port']) - # The following section is taken from https://github.com/django/django/blob/master/django/test/utils.py # This is a relatively simple XML comparator implementation based on Python's minidom library. @@ -467,4 +465,4 @@ def first_node(document): want_root = first_node(parseString(want)) got_root = first_node(parseString(got)) - return check_element(want_root, got_root) \ No newline at end of file + return check_element(want_root, got_root) diff --git a/requirements.txt b/requirements.txt index 39e7c76f6..463a1b9f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,6 @@ sphinx-rtd-theme pytest-bdd pytest-cov pytest-mock -pytest-capturelog pytest-twisted coverage pytest-runner