From edb937cab72549b621d22e941af9babf1fc48c32 Mon Sep 17 00:00:00 2001 From: Justin Mayfield Date: Tue, 4 Aug 2015 13:42:37 -0600 Subject: [PATCH 1/3] Py3K fix for -i option The string.rstrip method does not exist in Py3K. --- pycscope/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pycscope/__init__.py b/pycscope/__init__.py index 8a568bb..b2f0faf 100755 --- a/pycscope/__init__.py +++ b/pycscope/__init__.py @@ -20,7 +20,7 @@ -f reffile Use 'reffile' as cross-ref file name instead of 'cscope.out' -i srclistfile Use the contents of 'srclistfile' as the list of source files to scan""" -import getopt, sys, os, string, re +import getopt, sys, os, re import keyword, parser, symbol, token @@ -117,7 +117,8 @@ def main(argv=None): if o == "-f": indexfn = a if o == "-i": - args.extend(list(map(string.rstrip, open(a, 'r').readlines()))) + with open(a) as f: + args.extend(x.rstrip() for x in f) # Search current dir by default if len(args) == 0: From dbb2e220a7fdfb447d53e8587b541623f3e535d7 Mon Sep 17 00:00:00 2001 From: Justin Mayfield Date: Tue, 4 Aug 2015 17:15:44 -0600 Subject: [PATCH 2/3] Fix for unicode handling in Py3K. Py3K does not automatically encode unicode into its bytes repr so the len() call used for calculating an offset in the header is wrong. This patch does a simple py3k duck test and calls str.encode() when appropriate. --- pycscope/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycscope/__init__.py b/pycscope/__init__.py index b2f0faf..d6dd7b5 100755 --- a/pycscope/__init__.py +++ b/pycscope/__init__.py @@ -149,7 +149,7 @@ def writeIndex(basepath, fout, indexbuff, fnamesbuff): """ # Write the header and index index = ''.join(indexbuff) - index_len = len(index) + index_len = len(index.encode() if isinstance(u'' , str) else index) hdr_len = len(basepath) + 25 fout.write("cscope 15 %s -c %010d" % (basepath, hdr_len + index_len)) fout.write(index) From 2445e6a4b25faee975be7f312feee5334c679f86 Mon Sep 17 00:00:00 2001 From: Justin Mayfield Date: Tue, 4 Aug 2015 23:59:22 -0600 Subject: [PATCH 3/3] Use PEP 263 encoding detection in Py3k. For Py3K we can use tokenize.open to read source code files in the preferred style (pep 263). Trap any exception during source code parsing and continue on instead of aborting the process. For large file lists that include the python standard library (and its tests) there are unavoidable exceptions of a variety of sources and we don't want to break the entire DB generation when this happens. --- pycscope/__init__.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/pycscope/__init__.py b/pycscope/__init__.py index d6dd7b5..fc4bc1d 100755 --- a/pycscope/__init__.py +++ b/pycscope/__init__.py @@ -22,6 +22,7 @@ import getopt, sys, os, re import keyword, parser, symbol, token +import tokenize class Mark(object): @@ -176,7 +177,8 @@ def work(basepath, gen, debug): indexbuff_len = parseFile(basepath, fname, indexbuff, indexbuff_len, fnamesbuff, dump=debug) except (SyntaxError, AssertionError) as e: print("pycscope.py: %s: Line %s: %s" % (e.filename, e.lineno, e)) - pass + except Exception as e: + print("pycscope.py: %s: %s" % (fname, e)) return indexbuff, fnamesbuff @@ -222,15 +224,9 @@ def parseFile(basepath, relpath, indexbuff, indexbuff_len, fnamesbuff, dump=Fals """ # Open the file and get the contents fullpath = os.path.join(basepath, relpath) - try: - f = open(fullpath, 'rU') - except IOError as e: - # Can't open a file, emit message and ignore - print("pycscope.py: %s" % e) - return indexbuff_len - filecontents = f.read() - f.close() - + bestopen = getattr(tokenize, 'open', open) + with bestopen(fullpath) as f: + filecontents = f.read() # Add the file mark to the index fnamesbuff.append(relpath) indexbuff.append("\n%s%s\n\n" % (Mark(Mark.FILE), relpath))