diff --git a/utilities/project_helpers/scripts/check-message-translations.py b/utilities/project_helpers/scripts/check-message-translations.py index 813d68092aa9..07e6a43b7050 100644 --- a/utilities/project_helpers/scripts/check-message-translations.py +++ b/utilities/project_helpers/scripts/check-message-translations.py @@ -1,47 +1,41 @@ #!/usr/bin/python # -*- coding: utf-8 -*- -## USAGE EXAMPLE: python check_messsage_translations.sh cs - -import sys +import argparse import os -import codecs -import re -from check_message_lib import find_language_file_name +from check_message_lib import find_language_file_name, get_js_keys, get_xml_keys + +arg_parser = argparse.ArgumentParser(description='Compare the XML and JS message keys for two languages.') +arg_parser.add_argument('lang', help='First language as a 2-letter code') +arg_parser.add_argument('-base', required=False, default='en', help='Second language as a 2-letter code (defaults to "en")') +arguments = arg_parser.parse_args() +language1 = arguments.lang +language2 = arguments.base script_directory = os.path.dirname(os.path.realpath(__file__)) os.chdir(script_directory) -language1 = sys.argv[1] -language2 = sys.argv[2] if len(sys.argv) > 2 else 'en' -dspace_script = 'dspace-l10n-check.py' +def compare_keys(file_name1, file_name2, keys_function): + print('\n\nComparing {} and {}:'.format(file_name1, file_name2)) + keys1 = keys_function(file_name1) + keys2 = keys_function(file_name2) + report_delta(file_name1, file_name2, keys2-keys1) + report_delta(file_name2, file_name1, keys1-keys2) + +def report_delta(file_name1, file_name2, keys): + if (len(keys) == 0): + print('\n Every key in {} is also in {}.'.format(file_name2, file_name1)) + else: + print('\n Present in ' + file_name2 + ' but missing in ' + file_name1 + ':') + for key in keys: + print(' ' + key) xml_file_name1 = find_language_file_name(language1, 'xml') xml_file_name2 = find_language_file_name(language2, 'xml') -os.system('python ' + dspace_script + ' ' + xml_file_name1 + ' ' + xml_file_name2) - -js_key_regexp = r'^\s*["\']([\w-]+?)["\']\s*:' -def find_js_keys(js_file_name): - js_file = codecs.open(js_file_name, 'r', 'UTF-8') - keys = set() - for line in js_file: - match = re.search(js_key_regexp, line.strip(), re.U) - if (match): - keys.add(match.group(1)) - return keys +compare_keys(xml_file_name1, xml_file_name2, get_xml_keys) js_file_name1 = find_language_file_name(language1, 'js') -js_keys1 = find_js_keys(js_file_name1) js_file_name2 = find_language_file_name(language2, 'js') -js_keys2 = find_js_keys(js_file_name2) - -print '\nPresent in ' + js_file_name2 + ' but missing in ' + js_file_name1 + ':' -for key in (js_keys2 - js_keys1): - print key - -print '\nPresent in ' + js_file_name1 + ' but missing in ' + js_file_name2 + ':' -for key in (js_keys1 - js_keys2): - print key - +compare_keys(js_file_name1, js_file_name2, get_js_keys) diff --git a/utilities/project_helpers/scripts/check-message-usages.py b/utilities/project_helpers/scripts/check-message-usages.py index 4b172cb30f5d..6e1d448f4311 100644 --- a/utilities/project_helpers/scripts/check-message-usages.py +++ b/utilities/project_helpers/scripts/check-message-usages.py @@ -1,17 +1,25 @@ #!/usr/bin/python +# -*- coding: utf-8 -*- -import sys +import argparse import subprocess import codecs import os import re import xml.etree.ElementTree as xml -from check_message_lib import find_language_file_name, root_directory +from check_message_lib import find_language_file_name, ROOT_DIRECTORY -language = sys.argv[1] +arg_parser = argparse.ArgumentParser(description='Check for usage of XML and JS message keys in code.') +arg_parser.add_argument('lang', help='Language (as a 2-letter code) of the messages file') +arguments = arg_parser.parse_args() +language = arguments.lang + +script_directory = os.path.dirname(os.path.realpath(__file__)) +os.chdir(script_directory) + +LINE_REGEXP = r'^(.+?):(.*)$' -line_regexp = r'^(.+?):(.*)$' def find_xml_prefixes_and_files(): @@ -19,13 +27,13 @@ def find_xml_prefixes_and_files(): grep_command = 'grep -R -P "[>\'\\"](' + '|'.join(prefixes) + ')\\." --include=*.java --include=*.xsl --include=*.xmap --include=*.xslt --include=input-forms.xml --exclude-dir=*/target/* *' prefix_regexp = "[>'\"]((?:" + "|".join(prefixes) + ")\..+?)[<'\"]" - os.chdir(root_directory) + os.chdir(ROOT_DIRECTORY) with open(os.devnull, 'w') as devnull: output = subprocess.check_output(grep_command, shell=True, stderr=devnull) output_lines = output.strip().split('\n') message_prefixes = set() for grep_line in output_lines: - line_match = re.search(line_regexp, grep_line, re.U) + line_match = re.search(LINE_REGEXP, grep_line, re.U) (file_name, line) = line_match.groups() match_tuples = re.findall(prefix_regexp, line, re.U) for match_tuple in match_tuples: @@ -66,12 +74,12 @@ def add_js_results(language, results): key = message_match.group(1) result = {'type':'js', 'match':'no', 'key':key, 'file_name':None, 'prefix':None} grep_command = 'grep -R -P "(\\\\$|jQuery)\\.i18n\._\\([\'\\"]' + key + '[\'\\"][),]" --include=*.js --include=*.html --exclude-dir=*/target/* *' - os.chdir(root_directory) + os.chdir(ROOT_DIRECTORY) try: with open(os.devnull, 'w') as devnull: output = subprocess.check_output(grep_command, shell=True, stderr=devnull) output_lines = output.strip().split('\n') - line_match = re.search(line_regexp, output_lines[0], re.U) + line_match = re.search(LINE_REGEXP, output_lines[0], re.U) (file_name, line) = line_match.groups() result['match'] = 'full' result['file_name'] = file_name diff --git a/utilities/project_helpers/scripts/check_message_lib.py b/utilities/project_helpers/scripts/check_message_lib.py index 41b05de403d8..0e3e39b325c7 100644 --- a/utilities/project_helpers/scripts/check_message_lib.py +++ b/utilities/project_helpers/scripts/check_message_lib.py @@ -2,38 +2,54 @@ # -*- coding: utf-8 -*- import os +import re import codecs +import xml.etree.ElementTree as etree -script_directory = os.path.dirname(os.path.realpath(__file__)) -root_directory = script_directory + '/../../..' -xml_i18n_directory = root_directory + '/dspace/modules/xmlui/src/main/webapp/i18n' -js_i18n_directory = root_directory + '/dspace-xmlui/src/main/webapp/themes/UFAL/lib/js/messages' -xml_en_joint_file_name = '/tmp/messages-en.xml' +SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) +ROOT_DIRECTORY = SCRIPT_DIRECTORY + '/../../..' +XML_I18N_DIRECTORY = ROOT_DIRECTORY + '/dspace/modules/xmlui/src/main/webapp/i18n' +JS_I18N_DIRECTORY = ROOT_DIRECTORY + '/dspace-xmlui/src/main/webapp/themes/UFAL/lib/js/messages' +XML_EN_JOINT_FILE_NAME = '/tmp/messages-en.xml' +JS_KEY_REGEXP = r'^\s*["\']([\w-]+?)["\']\s*:' def find_language_file_name(language, kind): if (kind == 'xml'): if (language != 'en'): - file_name = xml_i18n_directory + '/messages_' + language + '.xml' + file_name = XML_I18N_DIRECTORY + '/messages_' + language + '.xml' else: - file_name = xml_en_joint_file_name + file_name = XML_EN_JOINT_FILE_NAME _create_xml_en_joint_file() elif (kind == 'js'): if (language != 'en'): - file_name = js_i18n_directory + '/messages_' + language + '.js' + file_name = JS_I18N_DIRECTORY + '/messages_' + language + '.js' else: - file_name = js_i18n_directory + '/messages.js' + file_name = JS_I18N_DIRECTORY + '/messages.js' return os.path.abspath(file_name) +def get_xml_keys(messages_file_name): + root = etree.parse(messages_file_name).getroot() + return {message.get('key') for message in root} + +def get_js_keys(js_file_name): + js_file = codecs.open(js_file_name, 'r', 'UTF-8') + keys = set() + for line in js_file: + match = re.search(JS_KEY_REGEXP, line.strip(), re.U) + if (match): + keys.add(match.group(1)) + return keys + ## Merge together all messages.xml into one temporary messages-en.xml. ## Avoids xml parsing to prevent namespace complications. def _create_xml_en_joint_file(): en_file_names = set() - for (dpath, dnames, fnames) in os.walk(root_directory): + for (dpath, dnames, fnames) in os.walk(ROOT_DIRECTORY): for fname in [os.path.join(dpath, fname) for fname in fnames]: if ('/target/' not in fname and fname.endswith('/messages.xml')): en_file_names.add(os.path.abspath(fname)) - print 'Constructing temporary /tmp/messages_en.xml from all messages.xml:\n ' + '\n '.join(en_file_names) + '\n' - en_joint_file = codecs.open(xml_en_joint_file_name, 'w', 'UTF-8') + print('\nConstructing temporary joint xml ' + XML_EN_JOINT_FILE_NAME + ' from all English messages.xml:\n ' + '\n '.join(en_file_names)) + en_joint_file = codecs.open(XML_EN_JOINT_FILE_NAME, 'w', 'UTF-8') for (index, en_file_name) in enumerate(en_file_names): en_file = codecs.open(en_file_name, 'r', 'UTF-8') if (index == 0): @@ -54,3 +70,5 @@ def _create_xml_en_joint_file(): en_file.close() en_joint_file.write('\n') en_joint_file.close() + + diff --git a/utilities/project_helpers/scripts/dspace-l10n-check.py b/utilities/project_helpers/scripts/dspace-l10n-check.py deleted file mode 100644 index ad914378af1d..000000000000 --- a/utilities/project_helpers/scripts/dspace-l10n-check.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python - -import sys -import collections - - -KEY, PREV, NEXT = range(3) - -class OrderedSet(collections.MutableSet): - - def __init__(self, iterable=None): - self.end = end = [] - end += [None, end, end] # sentinel node for doubly linked list - self.map = {} # key --> [key, prev, next] - if iterable is not None: - self |= iterable - - def __len__(self): - return len(self.map) - - def __contains__(self, key): - return key in self.map - - def add(self, key): - if key not in self.map: - end = self.end - curr = end[PREV] - curr[NEXT] = end[PREV] = self.map[key] = [key, curr, end] - - def discard(self, key): - if key in self.map: - key, prev, next = self.map.pop(key) - prev[NEXT] = next - next[PREV] = prev - - def __iter__(self): - end = self.end - curr = end[NEXT] - while curr is not end: - yield curr[KEY] - curr = curr[NEXT] - - def __reversed__(self): - end = self.end - curr = end[PREV] - while curr is not end: - yield curr[KEY] - curr = curr[PREV] - - def pop(self, last=True): - if not self: - raise KeyError('set is empty') - key = next(reversed(self)) if last else next(iter(self)) - self.discard(key) - return key - - def __repr__(self): - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, list(self)) - - def __eq__(self, other): - if isinstance(other, OrderedSet): - return len(self) == len(other) and list(self) == list(other) - return set(self) == set(other) - - def __del__(self): - self.clear() # remove circular references - - -class MessagesXmlParser(): - def __init__(self, filename): - import xml.etree.ElementTree as etree - - self.keys = [] - - tree = etree.parse(filename) - root = tree.getroot() - for message in root: - self.keys.append(message.attrib['key']) - -class MessagesPropertiesParser(): - def __init__(self, filename): - try: - import jprops - except: - print('Error: jprops module for parsing .properties files is missing. Download and follow installation instructions from http://mgood.github.com/jprops/') - sys.exit(2) - - self.keys = [] - - with open(filename) as fp: - for key, value in jprops.iter_properties(fp): - self.keys.append(key) - -if __name__ == "__main__": - if len(sys.argv) != 3: - print("Usage:") - print(" %s messages.xml messages_XX.xml" % (sys.argv[0])) - print("or") - print(" %s Messages.properties Messages_XX.properties" % (sys.argv[0])) - sys.exit(1) - - testfile = open(sys.argv[1], 'rb') - if testfile.readline().find('