diff --git a/CHANGELOG b/CHANGELOG index 8746aac..86db7ab 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,4 +1,6 @@ -* Thu Nov 26 2015 Sundeep Anand +* Wed Dec 02 2015 Sundeep Anand +- Bug 1215274 - specify minimum percentage completion on pull +- Rename zanatalib/project.py to zanatalib/projectutils.py - Bug 1156236 - use locale aliases defined in the server - added ProjectContext, Improved help, fixed code issues - added and in zanata.xml diff --git a/test/test_zanatacmd.py b/test/test_zanatacmd.py index 8d116ff..0e3d9ca 100644 --- a/test/test_zanatacmd.py +++ b/test/test_zanatacmd.py @@ -31,7 +31,7 @@ from zanataclient.zanatacmd import ZanataCommand from zanataclient.zanatalib import ZanataResource -from zanataclient.zanatalib.project import Project +from zanataclient.zanatalib.projectutils import Project class ZanataCmdTest(unittest.TestCase): diff --git a/zanataclient/cmdbase.py b/zanataclient/cmdbase.py index 115d8a3..7e7fb0d 100644 --- a/zanataclient/cmdbase.py +++ b/zanataclient/cmdbase.py @@ -61,6 +61,12 @@ def generate_zanatacmd(self, url, headers): sys.exit(1) return ZanataCommand(url, headers) + def check_essential(self, item, message): + if not item: + log.error(message) + sys.exit(1) + return item + class ListProjects(CommandsBase): def __init__(self, *args, **kargs): @@ -236,11 +242,7 @@ def get_files(self): filelist = [] tmlfolder = "" project_type = self.context_data.get('project_type') - - if not project_type: - log.error("The project type is unknown") - sys.exit(1) - elif project_type != 'podir' and project_type != 'gettext': + if project_type != 'podir' and project_type != 'gettext': log.error("The project type is not correct, please use 'podir' and 'gettext' as project type") sys.exit(1) @@ -425,8 +427,16 @@ def get_importparam(self, project_type, folder): import_param['project_type'] = project_type return import_param - def log_message(self, project_id, version_id, username): - log.info("Project: %s" % project_id) - log.info("Version: %s" % version_id) + def log_message(self, project_id, project_version, username): + log.info("Project: %s" % self.check_essential( + project_id, "Please specify PROJECT_ID with --project-id option or using zanata.xml" + )) + log.info("Version: %s" % self.check_essential( + project_version, "Please specify PROJECT_VERSION with --project-version option or using zanata.xml" + )) + log.info("Project Type: %s" % self.check_essential( + self.context_data.get('project_type'), + "Please specify PROJECT_TYPE with --project-type option or using zanata.xml" + )) log.info("Username: %s" % username) log.info("Source language: en-US") diff --git a/zanataclient/parseconfig.py b/zanataclient/parseconfig.py index e03f163..8e9cb7f 100644 --- a/zanataclient/parseconfig.py +++ b/zanataclient/parseconfig.py @@ -122,7 +122,8 @@ def read_project_config(self, filename): node = xmldoc.getElementsByTagName("trans-dir")[0] project_config['transdir'] = getCombinedTextChildren(node) - return project_config + return dict((node, value.strip() if isinstance(value, str) else value) + for node, value in project_config.items() if value) def getCombinedTextChildren(node): diff --git a/zanataclient/pullcmd.py b/zanataclient/pullcmd.py index 54ad45f..8b3183a 100644 --- a/zanataclient/pullcmd.py +++ b/zanataclient/pullcmd.py @@ -49,12 +49,7 @@ def run(self): sys.exit(1) locale_map = self.context_data.get('locale_map') - - if self.context_data.has_key('project_type'): - command_type = self.context_data.get('project_type') - else: - log.error("The project type is unknown") - sys.exit(1) + command_type = self.context_data.get('project_type') if self.context_data.get('publican_po'): # Keep dir option for publican/po pull @@ -76,5 +71,7 @@ def run(self): skeletons = False outpath = self.create_outpath(output_folder) - - self.zanatacmd.pull_command(locale_map, self.project_id, self.version_id, filelist, lang_list, outpath, command_type, skeletons) + filedict = self.zanatacmd.get_project_translation_stats( + self.project_id, self.version_id, self.context_data['mindocpercent'], lang_list, locale_map + ) if self.context_data.get('mindocpercent') else {file: lang_list for file in filelist} + self.zanatacmd.pull_command(locale_map, self.project_id, self.version_id, filedict, outpath, command_type, skeletons) diff --git a/zanataclient/zanata.py b/zanataclient/zanata.py index 4de36ba..17210ca 100644 --- a/zanataclient/zanata.py +++ b/zanataclient/zanata.py @@ -263,6 +263,13 @@ metavar='PUSHTYPE', ), ], + 'mindocpercent': [ + dict( + type='command', + long=['--min-doc-percent'], + metavar='MINDOCPERCENT', + ), + ], 'disablesslcert': [ dict( type='command', @@ -596,6 +603,8 @@ def pull(command_options, args, project_type=None): --project-version : id of the version (defaults to zanata.xml value) --transdir : translations will be written to this folder --lang : language list (defaults to zanata.xml locales) + --min-doc-percent : Only pull translation documents that have at least this percentage of messages translated. + Accepts an integer from 0 to 100. --noskeletons : omit po files when translations not found --disable-ssl-cert disable ssl certificate validation in 0.7.x python-httplib2 """ diff --git a/zanataclient/zanatacmd.py b/zanataclient/zanatacmd.py index 4a4f438..ded8375 100644 --- a/zanataclient/zanatacmd.py +++ b/zanataclient/zanatacmd.py @@ -27,8 +27,9 @@ from csvconverter import CSVConverter from zanatalib.resource import ZanataResource from zanatalib.glossaryservice import GlossaryService -from zanatalib.project import Project -from zanatalib.project import Iteration +from zanatalib.projectutils import ( + Project, Iteration, Stats +) from zanatalib.logger import Logger from zanatalib.error import ZanataException from zanatalib.error import NoSuchProjectException @@ -191,8 +192,8 @@ def list_projects(self): if not projects: # As we are catching exceptions related to reaching server, - # we may be certain that there is NO projects created. - self.log.error("There is no projects on the server.") + # we may be certain that there is NO project created. + self.log.info("There are no projects on this server.") sys.exit(1) for project in projects: @@ -335,7 +336,7 @@ def push_trans_command(self, transfolder, project_id, iteration_id, lang_list, l else: lang = item - self.log.info("\nPushing %s translation for %s to server:" % (item, project_id)) + self.log.info("Pushing %s translation for %s to server:" % (item, project_id)) if project_type == "podir": folder = os.path.join(transfolder, item) @@ -366,7 +367,7 @@ def push_trans_command(self, transfolder, project_id, iteration_id, lang_list, l name = filename + '.po' pofile = publicanutil.get_pofile_path(folder, name) - self.log.info("\nPushing the %s translation of %s to server:" % (item, filename)) + self.log.info("Pushing the %s translation of %s to server:" % (item, filename)) if not pofile or not os.path.isfile(pofile): self.log.error("Can not find the %s translation for %s" % (item, filename)) @@ -390,7 +391,7 @@ def push_command(self, file_list, srcfolder, project_id, iteration_id, copytrans publicanutil = PublicanUtility() for filepath in file_list: - self.log.info("\nPushing the content of %s to server:" % filepath) + self.log.info("Pushing the content of %s to server:" % filepath) plural_exist = publicanutil.check_plural(filepath) if plural_exist and not plural_support: self.log.error("The plural is only supported in zanata server >= 1.6, this file will be ignored") @@ -422,7 +423,7 @@ def push_command(self, file_list, srcfolder, project_id, iteration_id, copytrans self.import_po(filename, transdir, project_id, iteration_id, lang_list, locale_map, merge, project_type) - def pull_command(self, locale_map, project_id, iteration_id, filelist, lang_list, output, project_type, skeletons): + def pull_command(self, locale_map, project_id, iteration_id, filedict, output, project_type, skeletons): """ Retrieve the content of documents in a Project version from Zanata server. If the name of publican file is specified, the content of that file will be pulled from server. Otherwise, all the document of that @@ -431,7 +432,7 @@ def pull_command(self, locale_map, project_id, iteration_id, filelist, lang_list """ publicanutil = PublicanUtility() # if file no specified, retrieve all the files of project - for file_item in filelist: + for file_item, lang_list in filedict.items(): pot = "" result = "" folder = "" @@ -444,7 +445,7 @@ def pull_command(self, locale_map, project_id, iteration_id, filelist, lang_list name = file_item request_name = file_item - self.log.info("\nFetching the content of %s from Zanata server: " % name) + self.log.info("Fetching the content of %s from Zanata server" % name) try: pot = self.zanata_resource.documents.retrieve_template(project_id, iteration_id, request_name) @@ -487,7 +488,7 @@ def pull_command(self, locale_map, project_id, iteration_id, filelist, lang_list else: pofile = os.path.join(outpath, save_name + '.po') - self.log.info("Retrieving %s translation from server:" % item) + self.log.info("Retrieving %s translation from server: " % item) try: result = self.zanata_resource.documents.retrieve_translation(lang, project_id, iteration_id, request_name, skeletons) @@ -544,3 +545,27 @@ def delete_glossary(self, lang=None): self.log.error(str(e)) else: self.log.info("Successfully delete the glossary terms on the server") + + def get_project_translation_stats(self, project_id, project_version, min_doc_percent, lang_list, locale_map): + doc_locales_dict = {} + try: + server_return = self.zanata_resource.stats.get_project_stats(project_id, project_version) + except ZanataException, e: + self.log.error(str(e)) + else: + percent_dict = Stats(server_return).trans_percent_dict + for doc, stat in percent_dict.items(): + disqualify_locales = [] + for locale, trans_percent in stat.items(): + if trans_percent < int(min_doc_percent): + disqualify_locales.append(locale) + disqualify_locales = [alias for alias, locale in locale_map.items() + for lang in disqualify_locales if lang == locale] + if disqualify_locales: + self.log.info('Translation file for document %s for locales [%s] are skipped ' + 'because they are less than %s%% translated (--min-doc-percent setting)' % + (doc, ', '.join(map(str, disqualify_locales)), min_doc_percent)) + qualify_lang_set = set(lang_list) - set(disqualify_locales) + doc_locales_dict.update({doc: list(qualify_lang_set)}) + finally: + return doc_locales_dict diff --git a/zanataclient/zanatalib/__init__.py b/zanataclient/zanatalib/__init__.py index 72f448c..d3db984 100644 --- a/zanataclient/zanatalib/__init__.py +++ b/zanataclient/zanatalib/__init__.py @@ -24,6 +24,6 @@ from docservice import * from error import * from projectservice import * -from project import * +from projectutils import * from versionservice import * from logger import * diff --git a/zanataclient/zanatalib/logger.py b/zanataclient/zanatalib/logger.py index 8730f0c..db8568d 100644 --- a/zanataclient/zanatalib/logger.py +++ b/zanataclient/zanatalib/logger.py @@ -26,11 +26,11 @@ class Logger: def __init__(self): - self.enable_infoprefix = False + self.enable_infoprefix = True self.enable_warnprefix = True self.enable_errprefix = True - self.warn_prefix = 'warning: ' - self.error_prefix = 'error: ' + self.warn_prefix = '[WARN] ' + self.error_prefix = '[ERROR] ' self.info_prefix = '[INFO] ' def info(self, message): diff --git a/zanataclient/zanatalib/projectservice.py b/zanataclient/zanatalib/projectservice.py index 9a205f9..9a67c22 100644 --- a/zanataclient/zanatalib/projectservice.py +++ b/zanataclient/zanatalib/projectservice.py @@ -25,8 +25,8 @@ ) -from project import Project -from project import Iteration +from projectutils import Project +from projectutils import Iteration from service import Service diff --git a/zanataclient/zanatalib/project.py b/zanataclient/zanatalib/projectutils.py similarity index 51% rename from zanataclient/zanatalib/project.py rename to zanataclient/zanatalib/projectutils.py index 9aae297..6194fe6 100644 --- a/zanataclient/zanatalib/project.py +++ b/zanataclient/zanatalib/projectutils.py @@ -22,7 +22,7 @@ __all__ = ( - "Project", "Iteration" + "Project", "Iteration", "Stats" ) @@ -53,3 +53,46 @@ def set_iteration(self, iterations): def get_iteration(self, version_id): project_id = getattr(self, 'id') return self.__iterations.get(project_id, version_id) + + +class Stats(object): + def __init__(self, stats): + self.stats_dict = stats + + def _get_doc_trans_percent(self, doc_name, stats_dict): + trans_percent = {} + for stat in stats_dict: + if stat.get('locale'): + trans_percent.update({ + stat['locale']: int((float(stat.get('translated', 0) * 100) / + float(stat.get('total', 0)))) + }) + return {doc_name: trans_percent} + + @property + def project_version(self): + return self.stats_dict.get('id') + + @property + def trans_stats_dict(self): + return self.stats_dict.get('stats') + + @property + def trans_percent_dict(self): + trans_percent = {} + detailed_stats = self.stats_dict.get('detailedStats') + if isinstance(detailed_stats, list): + for doc in detailed_stats: + if isinstance(doc, dict) and doc.get('id') and doc.get('stats'): + trans_percent.update(self._get_doc_trans_percent(doc['id'], doc['stats'])) + return trans_percent + + @property + def trans_stats_detail_dict(self): + trans_dict = {} + detailed_stats = self.stats_dict.get('detailedStats') + if isinstance(detailed_stats, list): + for doc in detailed_stats: + if isinstance(doc, dict) and doc.get('id') and doc.get('stats'): + trans_dict.update({doc['id']: doc['stats']}) + return trans_dict diff --git a/zanataclient/zanatalib/resource.py b/zanataclient/zanatalib/resource.py index 1298241..a38cc45 100644 --- a/zanataclient/zanatalib/resource.py +++ b/zanataclient/zanatalib/resource.py @@ -29,6 +29,7 @@ from projectservice import ProjectService from versionservice import VersionService from glossaryservice import GlossaryService +from statservice import StatService class ZanataResource: @@ -38,6 +39,7 @@ def __init__(self, base_url, http_headers): self.documents = DocumentService(self.projects, base_url, http_headers) self.version = VersionService(base_url, http_headers) self.glossary = GlossaryService(base_url, http_headers) + self.stats = StatService(base_url, http_headers) def disable_ssl_cert_validation(self): self.projects.disable_ssl_cert_validation() diff --git a/zanataclient/zanatalib/rest/config.py b/zanataclient/zanatalib/rest/config.py index 825edd2..456839c 100644 --- a/zanataclient/zanatalib/rest/config.py +++ b/zanataclient/zanatalib/rest/config.py @@ -140,7 +140,15 @@ }, }, }, - 'StatisticsResource': {}, + 'StatisticsResource': { + '/stats/proj/{projectSlug}/iter/{iterationSlug}': { + http_methods[0]: { + 'path_params': ('projectSlug', 'iterationSlug'), + 'query_params': None, + 'response_media_type': media_types[0], + }, + } + }, 'TranslatedDocResource': { '/projects/p/{projectSlug}/iterations/i/{iterationSlug}/r/{id}/translations/{locale}': { http_methods[0]: { @@ -193,6 +201,7 @@ http_methods[0]) iteration_locales = resource('ProjectIterationLocalesResource', resource_config_dict['ProjectIterationLocalesResource'].keys()[0], http_methods[0]) +proj_trans_stats = resource('StatisticsResource', resource_config_dict['StatisticsResource'].keys()[0], http_methods[0]) # zanata-python-client operates on services listed here zpc_services = { 'server_version': server_version, @@ -212,6 +221,7 @@ 'commit_translation': commit_translation, 'project_locales': project_locales, 'iteration_locales': iteration_locales, + 'proj_trans_stats': proj_trans_stats, } diff --git a/zanataclient/zanatalib/statservice.py b/zanataclient/zanatalib/statservice.py new file mode 100644 index 0000000..b0b8c14 --- /dev/null +++ b/zanataclient/zanatalib/statservice.py @@ -0,0 +1,46 @@ +# vim:set et sts=4 sw=4: +# +# Zanata Python Client +# +# Copyright (c) 2015 Sundeep Anand +# Copyright (c) 2015 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. + + +__all__ = ( + "StatService", +) + +from service import Service + + +class StatService(Service): + _fields = ['base_url', 'http_headers'] + + def __init__(self, *args, **kargs): + super(StatService, self).__init__(*args, **kargs) + + def disable_ssl_cert_validation(self): + self.restclient.disable_ssl_cert_validation() + + def get_project_stats(self, project_id, project_version): + ext = "?detail=true&word=false" + res, content = self.restclient.process_request( + 'proj_trans_stats', project_id, project_version, + headers=self.http_headers, extension=ext + ) + return self.messages(res, content)