Merge pull request #31 from sundeep-co-in/STAGE

ZNTA-780: zanata min doc percent

CHANGELOG

@@ -1,4 +1,6 @@
-* Thu Nov 26 2015 Sundeep Anand <[email protected]>
+* Wed Dec 02 2015 Sundeep Anand <[email protected]>
+- Bug 1215274 - specify minimum percentage completion on pull
+- Rename zanatalib/project.py to zanatalib/projectutils.py
 - Bug 1156236 - use locale aliases defined in the server
 - added ProjectContext, Improved help, fixed code issues
 - added <src-dir> and <trans-dir> in zanata.xml

test/test_zanatacmd.py

@@ -31,7 +31,7 @@
 
 from zanataclient.zanatacmd import ZanataCommand
 from zanataclient.zanatalib import ZanataResource
-from zanataclient.zanatalib.project import Project
+from zanataclient.zanatalib.projectutils import Project
 
 
 class ZanataCmdTest(unittest.TestCase):

zanataclient/cmdbase.py

@@ -61,6 +61,12 @@ def generate_zanatacmd(self, url, headers):
                 sys.exit(1)
         return ZanataCommand(url, headers)
 
+    def check_essential(self, item, message):
+        if not item:
+            log.error(message)
+            sys.exit(1)
+        return item
+
 
 class ListProjects(CommandsBase):
     def __init__(self, *args, **kargs):
@@ -236,11 +242,7 @@ def get_files(self):
         filelist = []
         tmlfolder = ""
         project_type = self.context_data.get('project_type')
-
-        if not project_type:
-            log.error("The project type is unknown")
-            sys.exit(1)
-        elif project_type != 'podir' and project_type != 'gettext':
+        if project_type != 'podir' and project_type != 'gettext':
             log.error("The project type is not correct, please use 'podir' and 'gettext' as project type")
             sys.exit(1)
 
@@ -425,8 +427,16 @@ def get_importparam(self, project_type, folder):
         import_param['project_type'] = project_type
         return import_param
 
-    def log_message(self, project_id, version_id, username):
-        log.info("Project: %s" % project_id)
-        log.info("Version: %s" % version_id)
+    def log_message(self, project_id, project_version, username):
+        log.info("Project: %s" % self.check_essential(
+            project_id, "Please specify PROJECT_ID with --project-id option or using zanata.xml"
+        ))
+        log.info("Version: %s" % self.check_essential(
+            project_version, "Please specify PROJECT_VERSION with --project-version option or using zanata.xml"
+        ))
+        log.info("Project Type: %s" % self.check_essential(
+            self.context_data.get('project_type'),
+            "Please specify PROJECT_TYPE with --project-type option or using zanata.xml"
+        ))
         log.info("Username: %s" % username)
         log.info("Source language: en-US")

zanataclient/parseconfig.py

@@ -122,7 +122,8 @@ def read_project_config(self, filename):
             node = xmldoc.getElementsByTagName("trans-dir")[0]
             project_config['transdir'] = getCombinedTextChildren(node)
 
-        return project_config
+        return dict((node, value.strip() if isinstance(value, str) else value)
+                    for node, value in project_config.items() if value)
 
 
 def getCombinedTextChildren(node):

zanataclient/pullcmd.py

@@ -49,12 +49,7 @@ def run(self):
             sys.exit(1)
 
         locale_map = self.context_data.get('locale_map')
-
-        if self.context_data.has_key('project_type'):
-            command_type = self.context_data.get('project_type')
-        else:
-            log.error("The project type is unknown")
-            sys.exit(1)
+        command_type = self.context_data.get('project_type')
 
         if self.context_data.get('publican_po'):
             # Keep dir option for publican/po pull
@@ -76,5 +71,7 @@ def run(self):
             skeletons = False
 
         outpath = self.create_outpath(output_folder)
-
-        self.zanatacmd.pull_command(locale_map, self.project_id, self.version_id, filelist, lang_list, outpath, command_type, skeletons)
+        filedict = self.zanatacmd.get_project_translation_stats(
+            self.project_id, self.version_id, self.context_data['mindocpercent'], lang_list, locale_map
+        ) if self.context_data.get('mindocpercent') else {file: lang_list for file in filelist}
+        self.zanatacmd.pull_command(locale_map, self.project_id, self.version_id, filedict, outpath, command_type, skeletons)

zanataclient/zanata.py

@@ -263,6 +263,13 @@
             metavar='PUSHTYPE',
         ),
     ],
+    'mindocpercent': [
+        dict(
+            type='command',
+            long=['--min-doc-percent'],
+            metavar='MINDOCPERCENT',
+        ),
+    ],
     'disablesslcert': [
         dict(
             type='command',
@@ -596,6 +603,8 @@ def pull(command_options, args, project_type=None):
         --project-version   : id of the version (defaults to zanata.xml value)
         --transdir          : translations will be written to this folder
         --lang              : language list (defaults to zanata.xml locales)
+        --min-doc-percent   : Only pull translation documents that have at least this percentage of messages translated.
+                                Accepts an integer from 0 to 100.
         --noskeletons       : omit po files when translations not found
         --disable-ssl-cert disable ssl certificate validation in 0.7.x python-httplib2
     """

zanataclient/zanatacmd.py

@@ -27,8 +27,9 @@
 from csvconverter import CSVConverter
 from zanatalib.resource import ZanataResource
 from zanatalib.glossaryservice import GlossaryService
-from zanatalib.project import Project
-from zanatalib.project import Iteration
+from zanatalib.projectutils import (
+    Project, Iteration, Stats
+)
 from zanatalib.logger import Logger
 from zanatalib.error import ZanataException
 from zanatalib.error import NoSuchProjectException
@@ -191,8 +192,8 @@ def list_projects(self):
 
         if not projects:
             # As we are catching exceptions related to reaching server,
-            # we may be certain that there is NO projects created.
-            self.log.error("There is no projects on the server.")
+            # we may be certain that there is NO project created.
+            self.log.info("There are no projects on this server.")
             sys.exit(1)
 
         for project in projects:
@@ -335,7 +336,7 @@ def push_trans_command(self, transfolder, project_id, iteration_id, lang_list, l
                 else:
                     lang = item
 
-            self.log.info("\nPushing %s translation for %s to server:" % (item, project_id))
+            self.log.info("Pushing %s translation for %s to server:" % (item, project_id))
 
             if project_type == "podir":
                 folder = os.path.join(transfolder, item)
@@ -366,7 +367,7 @@ def push_trans_command(self, transfolder, project_id, iteration_id, lang_list, l
                         name = filename + '.po'
                     pofile = publicanutil.get_pofile_path(folder, name)
 
-                self.log.info("\nPushing the %s translation of %s to server:" % (item, filename))
+                self.log.info("Pushing the %s translation of %s to server:" % (item, filename))
 
                 if not pofile or not os.path.isfile(pofile):
                     self.log.error("Can not find the %s translation for %s" % (item, filename))
@@ -390,7 +391,7 @@ def push_command(self, file_list, srcfolder, project_id, iteration_id, copytrans
         publicanutil = PublicanUtility()
 
         for filepath in file_list:
-            self.log.info("\nPushing the content of %s to server:" % filepath)
+            self.log.info("Pushing the content of %s to server:" % filepath)
             plural_exist = publicanutil.check_plural(filepath)
             if plural_exist and not plural_support:
                 self.log.error("The plural is only supported in zanata server >= 1.6, this file will be ignored")
@@ -422,7 +423,7 @@ def push_command(self, file_list, srcfolder, project_id, iteration_id, copytrans
 
                 self.import_po(filename, transdir, project_id, iteration_id, lang_list, locale_map, merge, project_type)
 
-    def pull_command(self, locale_map, project_id, iteration_id, filelist, lang_list, output, project_type, skeletons):
+    def pull_command(self, locale_map, project_id, iteration_id, filedict, output, project_type, skeletons):
         """
         Retrieve the content of documents in a Project version from Zanata server. If the name of publican
         file is specified, the content of that file will be pulled from server. Otherwise, all the document of that
@@ -431,7 +432,7 @@ def pull_command(self, locale_map, project_id, iteration_id, filelist, lang_list
         """
         publicanutil = PublicanUtility()
         # if file no specified, retrieve all the files of project
-        for file_item in filelist:
+        for file_item, lang_list in filedict.items():
             pot = ""
             result = ""
             folder = ""
@@ -444,7 +445,7 @@ def pull_command(self, locale_map, project_id, iteration_id, filelist, lang_list
                 name = file_item
                 request_name = file_item
 
-            self.log.info("\nFetching the content of %s from Zanata server: " % name)
+            self.log.info("Fetching the content of %s from Zanata server" % name)
 
             try:
                 pot = self.zanata_resource.documents.retrieve_template(project_id, iteration_id, request_name)
@@ -487,7 +488,7 @@ def pull_command(self, locale_map, project_id, iteration_id, filelist, lang_list
                 else:
                     pofile = os.path.join(outpath, save_name + '.po')
 
-                self.log.info("Retrieving %s translation from server:" % item)
+                self.log.info("Retrieving %s translation from server: " % item)
 
                 try:
                     result = self.zanata_resource.documents.retrieve_translation(lang, project_id, iteration_id, request_name, skeletons)
@@ -544,3 +545,27 @@ def delete_glossary(self, lang=None):
             self.log.error(str(e))
         else:
             self.log.info("Successfully delete the glossary terms on the server")
+
+    def get_project_translation_stats(self, project_id, project_version, min_doc_percent, lang_list, locale_map):
+        doc_locales_dict = {}
+        try:
+            server_return = self.zanata_resource.stats.get_project_stats(project_id, project_version)
+        except ZanataException, e:
+            self.log.error(str(e))
+        else:
+            percent_dict = Stats(server_return).trans_percent_dict
+            for doc, stat in percent_dict.items():
+                disqualify_locales = []
+                for locale, trans_percent in stat.items():
+                    if trans_percent < int(min_doc_percent):
+                        disqualify_locales.append(locale)
+                disqualify_locales = [alias for alias, locale in locale_map.items()
+                                      for lang in disqualify_locales if lang == locale]
+                if disqualify_locales:
+                    self.log.info('Translation file for document %s for locales [%s] are skipped '
+                                  'because they are less than %s%% translated (--min-doc-percent setting)' %
+                                  (doc, ', '.join(map(str, disqualify_locales)), min_doc_percent))
+                qualify_lang_set = set(lang_list) - set(disqualify_locales)
+                doc_locales_dict.update({doc: list(qualify_lang_set)})
+        finally:
+            return doc_locales_dict

zanataclient/zanatalib/__init__.py

@@ -24,6 +24,6 @@
 from docservice import *
 from error import *
 from projectservice import *
-from project import *
+from projectutils import *
 from versionservice import *
 from logger import *

zanataclient/zanatalib/logger.py

@@ -26,11 +26,11 @@
 
 class Logger:
     def __init__(self):
-        self.enable_infoprefix = False
+        self.enable_infoprefix = True
         self.enable_warnprefix = True
         self.enable_errprefix = True
-        self.warn_prefix = 'warning: '
-        self.error_prefix = 'error: '
+        self.warn_prefix = '[WARN] '
+        self.error_prefix = '[ERROR] '
         self.info_prefix = '[INFO] '
 
     def info(self, message):

zanataclient/zanatalib/projectservice.py

@@ -25,8 +25,8 @@
 )
 
 
-from project import Project
-from project import Iteration
+from projectutils import Project
+from projectutils import Iteration
 from service import Service
 
 

zanataclient/zanatalib/project.py → zanataclient/zanatalib/projectutils.py

@@ -22,7 +22,7 @@
 
 
 __all__ = (
-    "Project", "Iteration"
+    "Project", "Iteration", "Stats"
 )
 
 
@@ -53,3 +53,46 @@ def set_iteration(self, iterations):
     def get_iteration(self, version_id):
         project_id = getattr(self, 'id')
         return self.__iterations.get(project_id, version_id)
+
+
+class Stats(object):
+    def __init__(self, stats):
+        self.stats_dict = stats
+
+    def _get_doc_trans_percent(self, doc_name, stats_dict):
+        trans_percent = {}
+        for stat in stats_dict:
+            if stat.get('locale'):
+                trans_percent.update({
+                    stat['locale']: int((float(stat.get('translated', 0) * 100) /
+                                         float(stat.get('total', 0))))
+                })
+        return {doc_name: trans_percent}
+
+    @property
+    def project_version(self):
+        return self.stats_dict.get('id')
+
+    @property
+    def trans_stats_dict(self):
+        return self.stats_dict.get('stats')
+
+    @property
+    def trans_percent_dict(self):
+        trans_percent = {}
+        detailed_stats = self.stats_dict.get('detailedStats')
+        if isinstance(detailed_stats, list):
+            for doc in detailed_stats:
+                if isinstance(doc, dict) and doc.get('id') and doc.get('stats'):
+                    trans_percent.update(self._get_doc_trans_percent(doc['id'], doc['stats']))
+        return trans_percent
+
+    @property
+    def trans_stats_detail_dict(self):
+        trans_dict = {}
+        detailed_stats = self.stats_dict.get('detailedStats')
+        if isinstance(detailed_stats, list):
+            for doc in detailed_stats:
+                if isinstance(doc, dict) and doc.get('id') and doc.get('stats'):
+                    trans_dict.update({doc['id']: doc['stats']})
+        return trans_dict

zanataclient/zanatalib/resource.py

@@ -29,6 +29,7 @@
 from projectservice import ProjectService
 from versionservice import VersionService
 from glossaryservice import GlossaryService
+from statservice import StatService
 
 
 class ZanataResource:
@@ -38,6 +39,7 @@ def __init__(self, base_url, http_headers):
         self.documents = DocumentService(self.projects, base_url, http_headers)
         self.version = VersionService(base_url, http_headers)
         self.glossary = GlossaryService(base_url, http_headers)
+        self.stats = StatService(base_url, http_headers)
 
     def disable_ssl_cert_validation(self):
         self.projects.disable_ssl_cert_validation()

zanataclient/zanatalib/rest/config.py

@@ -140,7 +140,15 @@
             },
         },
     },
-    'StatisticsResource': {},
+    'StatisticsResource': {
+        '/stats/proj/{projectSlug}/iter/{iterationSlug}': {
+            http_methods[0]: {
+                'path_params': ('projectSlug', 'iterationSlug'),
+                'query_params': None,
+                'response_media_type': media_types[0],
+            },
+        }
+    },
     'TranslatedDocResource': {
         '/projects/p/{projectSlug}/iterations/i/{iterationSlug}/r/{id}/translations/{locale}': {
             http_methods[0]: {
@@ -193,6 +201,7 @@
                            http_methods[0])
 iteration_locales = resource('ProjectIterationLocalesResource',
                              resource_config_dict['ProjectIterationLocalesResource'].keys()[0], http_methods[0])
+proj_trans_stats = resource('StatisticsResource', resource_config_dict['StatisticsResource'].keys()[0], http_methods[0])
 # zanata-python-client operates on services listed here
 zpc_services = {
     'server_version': server_version,
@@ -212,6 +221,7 @@
     'commit_translation': commit_translation,
     'project_locales': project_locales,
     'iteration_locales': iteration_locales,
+    'proj_trans_stats': proj_trans_stats,
 }