From d570edff9ad0375df1a72e0f62d74e9ba7575b19 Mon Sep 17 00:00:00 2001 From: Victor Leikehman Date: Wed, 27 Jul 2016 17:53:00 +0300 Subject: [PATCH] merge --- smop/backend.py | 413 +++++++++-------- smop/callgraph.py | 54 +-- smop/core.py | 51 ++- smop/lexer.py | 120 +++-- smop/main.py | 234 +++------- smop/node.py | 59 ++- smop/options.py | 100 +++- smop/parse.py | 1029 +++++++++++++++++++++++------------------- smop/reord.py | 4 + smop/resolve.py | 230 +++++----- smop/test_lexer.py | 1 + smop/test_primes.py | 2 + smop/test_runtime.py | 132 ++++++ smop/test_solver.py | 34 ++ 14 files changed, 1364 insertions(+), 1099 deletions(-) create mode 100644 smop/reord.py create mode 100644 smop/test_primes.py create mode 100644 smop/test_runtime.py create mode 100644 smop/test_solver.py diff --git a/smop/backend.py b/smop/backend.py index 37ed86e6..b04a20e8 100644 --- a/smop/backend.py +++ b/smop/backend.py @@ -1,5 +1,5 @@ # smop -- Simple Matlab to Python compiler -# Copyright 2011-2014 Victor Leikehman +# Copyright 2011-2016 Victor Leikehman """ Calling conventions: @@ -29,6 +29,8 @@ ".^": "**", "./": "/", ".*": "*", + ".*=" : "*", + "./=" : "/", } def backend(t,*args): @@ -47,78 +49,34 @@ def _backend(self,level=0): #import pdb; pdb.set_trace() return "cat(%s)" % self.args[0]._backend() -@extend(node.cellarrayref) -def _backend(self,level=0): - return "%s[%s]" % (self.func_expr._backend(), - self.args._backend()) -@extend(node.cellarray) -def _backend(self,level=0): - return "cellarray([%s])" % self.args._backend() - -#@extend(node.concat_list) -#def _backend(self,level=0): -# return ";".join([t._backend() for t in self]) - -@extend(node.ravel) -def _backend(self,level=0): - return "%s.ravel()" % self.args[0]._backend() - -@extend(node.transpose) -def _backend(self,level=0): - return "%s.T" % self.args[0]._backend() - -@extend(node.expr_stmt) -def _backend(self,level=0): - return self.expr._backend() - -@extend(node.return_stmt) -def _backend(self,level=0): - if not self.ret: - return "return" - else: - return "return %s" % self.ret._backend() - -@extend(node.continue_stmt) -def _backend(self,level=0): - return "continue" +# Sometimes user's variable names in the matlab code collide with Python +# reserved words and constants. We handle this in the backend rather than in +# the lexer, to keep the target language separate from the lexer code. -@extend(node.global_stmt) -def _backend(self,level=0): - return "global %s" % self.global_list._backend() +# Some names, such as matlabarray, may collide with the user defined names. +# Both cases are solved by appending a trailing underscore to the user's names. -@extend(node.global_list) -def _backend(self,level=0): - return ",".join([t._backend() for t in self]) - -@extend(node.break_stmt) -def _backend(self,level=0): - return "break" - -@extend(node.string) -def _backend(self,level=0): - if self.value.find("'")+1: - return 'char("%s")' % (self.value) - else: - return "char('%s')" % (self.value) - -@extend(node.number) -def _backend(self,level=0): - #if type(self.value) == int: - # return "%s.0" % self.value - return str(self.value) +reserved = set( + """ + and assert break class continue + def del elif else except + exec finally for from global + if import in is lambda + not or pass print raise + return try while + + Data Float Int Numeric Oxphys + array close float int input + open range type write zeros + + len + """.split()) -@extend(node.logical) -def _backend(self,level=0): - if self.value == 0: - return "false" - else: - return "true" + #acos asin atan cos e + #exp fabs floor log log10 + #pi sin sqrt tan -# @extend(node.range) -# def _backend(self,level=0): -# i = node.ident.new("I") -# return "[ (%s, %s=%s,%s) ]" % (i,i,self.args[0],self.args[1]) - + @extend(node.add) def _backend(self,level=0): if (self.args[0].__class__ is node.number and @@ -128,12 +86,56 @@ def _backend(self,level=0): else: return "(%s+%s)" % (self.args[0]._backend(), self.args[1]._backend()) - -@extend(node.sub) + +@extend(node.arrayref) def _backend(self,level=0): - return "(%s-%s)" % (self.args[0]._backend(), - self.args[1]._backend()) - + fmt = "%s[%s]" + return fmt % (self.func_expr._backend(), + self.args._backend()) + +@extend(node.break_stmt) +def _backend(self,level=0): + return "break" + +@extend(node.builtins) +def _backend(self,level=0): + #if not self.ret: + return "%s(%s)" % (self.__class__.__name__, + self.args._backend()) + +@extend(node.cellarray) +def _backend(self,level=0): + return "cellarray([%s])" % self.args._backend() + +@extend(node.cellarrayref) +def _backend(self,level=0): + return "%s[%s]" % (self.func_expr._backend(), + self.args._backend()) + +@extend(node.comment_stmt) +def _backend(self,level=0): + s = self.value.strip() + if not s: + return "" + if s[0] in "%#": + return s.replace("%","#") + #return self.value + assert 0 + +@extend(node.concat_list) +def _backend(self,level=0): + #import pdb; pdb.set_trace() + return ",".join(["[%s]"%t._backend() for t in self]) + +@extend(node.continue_stmt) +def _backend(self,level=0): + return "continue" + +@extend(node.dot) +def _backend(self,level=0): + return "%s.dot(%s)" % (self.args[0]._backend(), + self.args[1]._backend()) + @extend(node.expr) def _backend(self,level=0): if self.op == '@': # FIXME @@ -195,21 +197,35 @@ def _backend(self,level=0): return ret+"%s(%s)" % (self.op, ",".join([t._backend() for t in self.args])) -@extend(node.arrayref) + +@extend(node.expr_list) def _backend(self,level=0): -# if (len(self.args) == 1 and not -# (self.args[0].__class__== node.expr and self.args[0].op=="::")): -# fmt = "%s(%s)" -# else: - if options.subscripts == "round": - fmt = "%s(%s)" - elif options.subscripts == "square": - fmt = "%s[%s]" - else: - assert False, options.subscripts - return fmt % (self.func_expr._backend(), - self.args._backend()) + return ",".join([t._backend() for t in self]) + +@extend(node.expr_stmt) +def _backend(self,level=0): + return self.expr._backend() + +@extend(node.for_stmt) +def _backend(self,level=0): + fmt = "for %s in %s.reshape(-1):%s" + return fmt % (self.ident._backend(), + self.expr._backend(), + self.stmt_list._backend(level+1)) + +@extend(node.func_stmt) +def _backend(self,level=-1): + if len(self.args) == 1 and self.args[0].name == "varargin": + s = "\ndef %s(*args):" % self.ident._backend() + s += "\n varargin = cellarray(args)" + return s + + s = "\ndef %s(%s*args,**kwargs):" + t = self.ident._backend() + u = ",".join(["%s=None" % a for a in self.args if isinstance(a,node.ident)]) + return s % (t, (u+",") if u else "") + @extend(node.funcall) def _backend(self,level=0): #import pdb; pdb.set_trace() @@ -224,96 +240,108 @@ def _backend(self,level=0): self.args._backend(), self.nargout) + +@extend(node.global_list) +def _backend(self,level=0): + return ",".join([t._backend() for t in self]) + +@extend(node.ident) +def _backend(self,level=0): + return self.name if self.name not in reserved else self.name+'_' + +@extend(node.if_stmt) +def _backend(self,level=0): + s = "if %s:%s" % (self.cond_expr._backend(), + self.then_stmt._backend(level+1)) + if self.else_stmt: + # Eech. This should have been handled in the parser. + if self.else_stmt.__class__ == node.if_stmt: + self.else_stmt = node.stmt_list([self.else_stmt]) + s += "\n"+indent*level + s += "else:%s" % self.else_stmt._backend(level+1) + return s + +@extend(node.lambda_expr) +def _backend(self,level=0): + return 'lambda %s: %s' % (self.args._backend(), + self.ret._backend()) + @extend(node.let) def _backend(self,level=0): -# if options.line_numbering: -# s = "# %d\n" % self.lineno + level*indent -# else: -# s = '' + if options.line_numbers: + s = "# %s:%d\n%s" % (options.filename, + self.lineno, + level*indent) + else: + s = '' #if self.args.__class__ is node.funcall: # self.args.nargout = self.nargout if self.ret.__class__ is node.expr and self.ret.op == "." : try: if self.ret.args[1].op == 'parens': - s = "setattr(%s,%s,%s)" % (self.ret.args[0]._backend(), + s += "setattr(%s,%s,%s)" % (self.ret.args[0]._backend(), self.ret.args[1].args[0]._backend(), self.args._backend()) except: - s = "%s%s = copy(%s)" % (self.ret.args[0]._backend(), + s += "%s%s = copy(%s)" % (self.ret.args[0]._backend(), self.ret.args[1]._backend(), self.args._backend()) elif (self.ret.__class__ is node.ident and self.args.__class__ is node.ident): - s = "%s=copy(%s)" % (self.ret._backend(), + s += "%s=copy(%s)" % (self.ret._backend(), self.args._backend()) else: - s = "%s=%s" % (self.ret._backend(), + s += "%s=%s" % (self.ret._backend(), self.args._backend()) return s - -@extend(node.expr_list) + +@extend(node.logical) def _backend(self,level=0): - return ",".join([t._backend() for t in self]) - -@extend(node.concat_list) + if self.value == 0: + return "false" + else: + return "true" + +@extend(node.matrix) def _backend(self,level=0): - #import pdb; pdb.set_trace() - return ",".join(["[%s]"%t._backend() for t in self]) - -# @extend(node.call_stmt) -# def _backend(self,level=0): -# return "CALL %s(%s,%s)" % (self.func_expr._backend(), -# self.args._backend(), -# self.ret._backend()) - -fortran_type = { - '@': '***', - 'd': 'DOUBLE PRECISION', - 'i': 'INTEGER', - 'l': 'LOGICAL', - 'c': 'CHARACTER', -} - -# def decl__backend(i): -# assert isinstance(i,ident) -# try: -# if i._rank() == 0: -# return "%s :: %s\n" % (fortran_type[i._type()], -# i) -# return ("%s,DIMENSION(%s),ALLOCATABLE :: %s\n" % -# (fortran_type[i._type()], -# ",".join([":" for j in range(i._rank())]), i)) -# except: -# return "??? :: %s\n" % i -@extend(node.function) -def _backend(self,level=0): - s = self.head._backend(level) - t = self.body._backend(level+1) - return "%s%s" % (s,t) - - -# Sometimes variable names collide with _python reserved -# words and constants. We handle this in the _backend rather than in -# the lexer, to keep the target language separate from -# the lexer code. -reserved = set( - """ - abs and apply as assert basestring bin bool break buffer bytearray - callable chr class classmethod cmp coerce compile complex continue copyright - credits def del delattr dict dir divmod elif Ellipsis else enumerate eval - except exec execfile exit False file filter finally float for format from - frozenset getattr global globals hasattr hash help hex id if import __import__ - in input int intern is isinstance issubclass iter lambda len license list - locals long map memoryview next None not not NotImplemented object oct - open or ord pass pow print property quit raise range raw_input reduce reload - repr return reversed round set setattr slice sorted staticmethod str sum super - True try tuple type unichr unicode vars while with xrange yield zip struct - """.split()) - -@extend(node.ident) + # TODO empty array has shape of 0 0 in matlab + # size([]) + # 0 0 + if not self.args: + return "[]" + elif any(a.__class__ is node.string for a in self.args): + return " + ".join(a._backend() for a in self.args) + else: + #import pdb; pdb.set_trace() + return "cat(%s)" % self.args[0]._backend() + +@extend(node.null_stmt) def _backend(self,level=0): - return self.name if self.name not in reserved else self.name+'_' + return "" + +@extend(node.number) +def _backend(self,level=0): + #if type(self.value) == int: + # return "%s.0" % self.value + return str(self.value) + +@extend(node.persistent_stmt) #FIXME +@extend(node.global_stmt) +def _backend(self,level=0): + return "global %s" % self.global_list._backend() + +@extend(node.ravel) +def _backend(self,level=0): + return "%s.ravel()" % self.args[0]._backend() + +@extend(node.return_stmt) +def _backend(self,level=0): + if not self.ret: + return "return" + else: + return "return %s" % self.ret._backend() + @extend(node.stmt_list) def _backend(self,level=0): sep = "\n"+indent*level @@ -322,55 +350,25 @@ def _backend(self,level=0): else: return sep+"pass" -@extend(node.func_decl) -def _backend(self,level=0): - if self.args and isinstance(self.args[-1],node.ident) and self.args[-1].name == "varargin": - del self.args[-1] - s = ",".join(["%s=None" % a for a in self.args if isinstance(a,node.ident)]+["*args,**kwargs"]) - s = "def %s(%s):\n" % (self.ident._backend(), s) - s += ' nargout = kwargs["nargout"] if kwargs else None\n' - s += ' varargin = cellarray(args)\n' - s += " nargin = %d-[%s].count(None)+len(args)\n" % (len(self.args), - ",".join([(a._backend() if isinstance(a,node.ident) else a.ret._backend()) - for a in self.args])) # sic: a.name - return s -""" -@extend(node.allocate_stmt) -def _backend(self,level=0): - s = "ALLOCATE (%s(%s))" % (self.ident._backend(), - self.args._backend()) - return s -""" - -@extend(node.lambda_expr) -def _backend(self,level=0): - return 'lambda %s: %s' % (self.args._backend(), - self.ret._backend()) -@extend(node.for_stmt) + +@extend(node.string) def _backend(self,level=0): - fmt = "for %s in %s.reshape(-1):%s" - return fmt % (self.ident._backend(), - self.expr._backend(), - self.stmt_list._backend(level+1)) - -@extend(node.if_stmt) +# s = self.value.strip() +# if not s: +# return "" +# if s[0] in "%#": +# return "\n"+s.replace("%","#") + return '"%s"' % self.value.encode("string_escape").replace('"','\\"') + +@extend(node.sub) def _backend(self,level=0): - s = "if %s:%s" % (self.cond_expr._backend(), - self.then_stmt._backend(level+1)) - if self.else_stmt: - # Eech. This should have been handled in the parser. - if self.else_stmt.__class__ == node.if_stmt: - self.else_stmt = node.stmt_list([self.else_stmt]) - s += "\n"+indent*level - s += "else:%s" % self.else_stmt._backend(level+1) - return s - -@extend(node.while_stmt) + return "(%s-%s)" % (self.args[0]._backend(), + self.args[1]._backend()) + +@extend(node.transpose) def _backend(self,level=0): - fmt = "while %s:\n%s\n" - return fmt % (self.cond_expr._backend(), - self.stmt_list._backend(level+1)) - + return "%s.T" % self.args[0]._backend() + @extend(node.try_catch) def _backend(self,level=0): fmt = "try:%s\n%sfinally:%s" @@ -378,15 +376,12 @@ def _backend(self,level=0): indent*level, self.finally_stmt._backend(level+1)) -@extend(node.builtins) + +@extend(node.while_stmt) def _backend(self,level=0): - #if not self.ret: - return "%s(%s)" % (self.__class__.__name__, - self.args._backend()) - # else: - # return ("%s=%s(%s)" % (self.ret._backend(), - # self.__class__.__name__, - # self.args._backend())) + fmt = "while %s:\n%s\n" + return fmt % (self.cond_expr._backend(), + self.stmt_list._backend(level+1)) ### @extend(node.strcmp) ### def _backend(self,level=0): diff --git a/smop/callgraph.py b/smop/callgraph.py index 6115adf3..350bf154 100644 --- a/smop/callgraph.py +++ b/smop/callgraph.py @@ -1,49 +1,31 @@ -import parse,sys +import parse,sys,os import networkx as nx -import node,resolve +import node,resolve,options -def callgraph(func_list): +def callgraph(G, stmt_list): """ Build callgraph of func_list, ignoring built-in functions """ - G = nx.DiGraph() - for func in func_list: - G.add_node(func.head.ident.name) + func_list = [] + for stmt in stmt_list: + try: + G.add_node(stmt.head.ident.name) + func_list.append(stmt) + except: + pass for func in func_list: assert isinstance(func,node.function) func_name = func.head.ident.name - resolve.resolve(func) + #resolve.resolve(func) for s in node.postorder(func): if (s.__class__ is node.funcall and - s.func_expr.__class__ is node.ident and - s.func_expr.name in G.nodes()): - G.add_edge(func_name,s.func_expr.name) - return G - -G = nx.DiGraph() - -def postorder_edge(u): - if isinstance(u,node.node): - for v in u: - for t in postorder_edge(v): - yield (v,t) - yield (u,u) # returns only traversible objects - -def foo(tree): - G = nx.DiGraph() - for u,v in postorder_edge(tree): - G.add_edge(id(u),id(v)) - return G - -def main(): - func_list = parse.parse(open(sys.argv[1]).read()) - G = foo(func_list) - #G = callgraph(func_list) - nx.write_dot(G,"G.dot") - #H = nx.dfs_tree(G,'solver') - #nx.write_dot(H,"H.dot") - #print nx.topological_sort(H) - + s.func_expr.__class__ is node.ident): + #if s.func_expr.name in G.nodes(): + G.add_edge(func_name,s.func_expr.name) + #nx.write_dot(G,"G.dot") + #for u in G.nodes(): + # if G.out_degree(u) == 0: + # print u if __name__ == '__main__': main() diff --git a/smop/core.py b/smop/core.py index 0cbbaf4b..10aa8dee 100644 --- a/smop/core.py +++ b/smop/core.py @@ -6,8 +6,7 @@ import __builtin__ import numpy -from sparsearray import sparsearray as sparse -from numpy import sqrt,eye +from numpy import sqrt from numpy.fft import fft2 from numpy.linalg import inv from numpy.linalg import qr as _qr @@ -54,6 +53,10 @@ class matlabarray(np.ndarray): """ >>> matlabarray() matlabarray([], shape=(0, 0), dtype=float64) + >>> matlabarray([arange(1,5), arange(1,5)]) + matlabarray([1, 2, 3, 4, 5, 1, 2, 3, 4, 5]) + >>> matlabarray(["hello","world"]) + matlabarray("helloworld") """ def __new__(cls,a=[],dtype=None): @@ -303,14 +306,16 @@ class char is a rectangular string matrix, which >>> print s hello world - - >>> s.shape - (2, 5) + >>> s=char([104, 101, 108, 108, 111, 119, 111, 114, 108, 100]) + >>> s.shape = 2,5 + >>> print s + hello + world """ def __new__(cls, a=""): if not isinstance(a,str): - raise NotImplementedError + a = "".join([chr(c) for c in a]) obj = np.array(list(a), dtype='|S1', copy=False, @@ -332,7 +337,7 @@ def __str__(self): return "\n".join("".join(s) for s in self) raise NotImplementedError -class struct_(object): +class struct(object): def __init__(self,*args): for i in range(0,len(args),2): setattr(self,str(args[i]),args[i+1]) @@ -356,7 +361,7 @@ def arange(start,stop,step=1,**kwargs): step, **kwargs).reshape(1,-1),**kwargs) def cat(*args): - return np.concatenate([matlabarray(a) for a in args],axis=1) + return matlabarray(np.concatenate([matlabarray(a) for a in args],axis=1)).reshape(-1) def ceil(a): return numpy.ceil(a) @@ -557,16 +562,27 @@ def rand(*args,**kwargs): except: pass -def rand(*args,**kwargs): - if not args: - return np.random.rand() - if len(args) == 1: - args += args - try: - return np.random.rand(np.prod(args)).reshape(args,order="F") - except: +def assert_(a,b=None): + if b is None: + assert a + else: + assert isequal(a,b) + +def shared(a): pass +def rand(*args,**kwargs): + """from core aka libsmop.py""" + return np.random.rand() + # if not args: + # return np.random.rand() + # if len(args) == 1: + # args += args + # try: + # return np.random.rand(np.prod(args)).reshape(args,order="F") + # except: + # pass + def randn(*args,**kwargs): if not args: return np.random.randn() @@ -665,6 +681,9 @@ def zeros(*args,**kwargs): args += args return matlabarray(np.zeros(args,**kwargs)) +def isa(a,b): + return True + if __name__ == "__main__": import doctest doctest.testmod() diff --git a/smop/lexer.py b/smop/lexer.py index 29f51574..6c7c0aa4 100644 --- a/smop/lexer.py +++ b/smop/lexer.py @@ -1,25 +1,26 @@ -# SMOP compiler -- Simple Matlab/Octave to Python compiler -# Copyright 2011-2014 Victor Leikehman +# SMOP -- Simple Matlab/Octave to Python compiler +# Copyright 2011-2016 Victor Leikehman import sys import re from zlib import adler32 -import lex -from lex import TOKEN +import ply.lex as lex +from ply.lex import TOKEN import readline +import options class IllegalCharacterError(Exception): pass -tokens = [ - "AND", "ANDAND", "ANDEQ", "BACKSLASH", "COLON", "COMMA", "DIV","DIVEQ", - "DOT", "DOTDIV", "DOTDIVEQ", "DOTEXP", "DOTMUL","DOTMULEQ", "END_EXPR", - "END_STMT", "EQ", "EQEQ", "EXP", "EXPEQ", "FIELD", "GE", "GT", "HANDLE", - "IDENT", "LBRACE", "LBRACKET", "LE", "LPAREN", "LT", - "MINUS","MINUSMINUS","MINUSEQ","MUL","MULEQ","NE", "NEG", - "NUMBER", "OR","OREQ", "OROR", "PLUS", "PLUSEQ","PLUSPLUS", - "RBRACE", "RBRACKET", "RPAREN", "SEMI", "STRING", "TRANSPOSE", -] +tokens = [ "AND", "ANDAND", "ANDEQ", "BACKSLASH", "COLON", "COMMA", + "DIV","DIVEQ", "DOT", "DOTDIV", "DOTDIVEQ", "DOTEXP", + "DOTMUL","DOTMULEQ", "END_EXPR", "END_STMT", "EQ", "EQEQ", + "EXP", "EXPEQ", "FIELD", "GE", "GT", "HANDLE", "IDENT", + "LBRACE", "LBRACKET", "LE", "LPAREN", "LT", + "MINUS","MINUSMINUS","MINUSEQ","MUL","MULEQ","NE", "NEG", + "NUMBER", "OR","OREQ", "OROR", "PLUS", "PLUSEQ","PLUSPLUS", + "RBRACE", "RBRACKET", "RPAREN", "SEMI", "STRING", + "TRANSPOSE", "ERROR_STMT", "COMMENT" ] reserved = { "break" : "BREAK", @@ -85,7 +86,8 @@ def new(): states = (("matrix","inclusive"), ("afterkeyword","exclusive")) - ws = r"(\s|(\#|%).*\n|\.\.\..*\n|\\\n)" + ws = r"(\s|\.\.\..*\n|\\\n)" + #ws = r"(\s|(\#|(%[^!])).*\n|\.\.\..*\n|\\\n)" ws1 = ws+"+" ws0 = ws+"*" ms = r"'([^']|(''))*'" @@ -94,11 +96,13 @@ def new(): id = r"[a-zA-Z_][a-zA-Z_0-9]*" def unescape(s): + """ + ffd52d5fc5 + """ if s[0] == "'": - return s[1:-1].replace("''","'") + return s[1:-1].replace("''","'").decode("string_escape") else: - return s[1:-1].replace('""','"') - #return s[1:-1].decode('string_escape').replace('""','"') + return s[1:-1].replace('""','"').decode("string_escape") @TOKEN(mos) def t_afterkeyword_STRING(t): @@ -133,26 +137,30 @@ def t_STRING(t): def t_IDENT(t): t.lexer.lineno += t.value.count("\n") if t.value[0] == ".": - # Reserved words are not reserved when used as fields. - # So return=1 is illegal, but foo.return=1 is fine. + # Reserved words are not reserved + # when used as fields. So return=1 + # is illegal, but foo.return=1 is fine. t.type = "FIELD" return t - if t.value in ("endwhile","endfunction","endif","endfor", - "endswitch","end_try_catch"): # octave + if t.value in ("endwhile","endfunction", + "endif","endfor", + "endswitch","end_try_catch"): t.type = "END_STMT" return t if t.value == "end": - if t.lexer.parens > 0 or t.lexer.brackets > 0 or t.lexer.braces > 0: + if (t.lexer.parens > 0 or + t.lexer.brackets > 0 or + t.lexer.braces > 0): t.type = "END_EXPR" else: t.type = "END_STMT" else: t.type = reserved.get(t.value,"IDENT") - if t.type != "IDENT" and t.lexer.lexdata[t.lexer.lexpos]=="'": + if (t.type != "IDENT" and + t.lexer.lexdata[t.lexer.lexpos]=="'"): t.lexer.begin("afterkeyword") return t - def t_LPAREN(t): r"\(" t.lexer.parens += 1 @@ -215,6 +223,8 @@ def t_SEMI(t): def t_NUMBER(t): r"(0x[0-9A-Fa-f]+)|((\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?[ij]?)" + # <-------------> <------------------><-------------> + # int,oct,hex float exp if t.value[-1] == 'i': t.value = t.value[:-1]+'j' t.value = eval(t.value) @@ -228,18 +238,25 @@ def t_NEWLINE(t): t.type = "SEMI" return t - def t_comment(t): - r"(%|\#).*" - pass + def t_ERROR_STMT(t): + r"%!(error|warning|test|demo).*" + return t + # keep multiline comments + def t_COMMENT(t): + r"(^[ \t]*[%#].*\n)+" + t.lexer.lineno += t.value.count("\n") + if not options.keep_comments: + return + t.type = "COMMENT" + return t + + # drop end-of-line comments + def t_comment(t): + r"(%|\#)!?" + if not options.testing_mode or t.value[-1] != "!": + t.lexer.lexpos = t.lexer.lexdata.find("\n",t.lexer.lexpos) -# @TOKEN(ws+r"(?=[-+]\S)") -# def t_matrix_WHITESPACE(t): -# #r"\s+(?=[-+]\S)" -# # Whitespace, followed by + or - followed by anything but whitespace -# t.lexer.lineno += t.value.count("\n") -# t.type = "COMMA" -# return t @TOKEN(r"(?<=\w)" + ws1 + r"(?=\()") def t_matrix_BAR(t): @@ -298,24 +315,33 @@ def t_error(t): column=t.lexer.lexpos - t.lexer.lexdata.rfind("\n",0,t.lexer.lexpos) raise IllegalCharacterError(t.lineno,column,t.value[0]) - lexer = lex.lex(reflags=re.I) + + lexer = lex.lex(reflags=re.MULTILINE) lexer.brackets = 0 # count open square brackets lexer.parens = 0 # count open parentheses lexer.braces = 0 # count open curly braces + lexer.stack = [] return lexer -if __name__ == "__main__": +def main(): lexer = new() - try: - while 1: - line = raw_input("=>> ") - if not line: - continue - while line[-1] == "\\": - line = line[:-1] + raw_input("... ") + line = "" + while 1: + try: + line += raw_input("=>> ").decode("string_escape") print len(line), [c for c in line] + except EOFError: + reload(sys.modules["lexer.py"]) lexer.input(line) - for tok in lexer: - print len(str(tok.value)), tok - except EOFError: - pass + print list(tok for tok in lexer) + line = "" + +if __name__ == "__main__": + options.testing_mode = 0 + options.debug_lexer = 0 + lexer = new() + buf = open(sys.argv[1]).read() + lexer.input(buf) + for tok in lexer: + print tok + diff --git a/smop/main.py b/smop/main.py index b8a4b953..61dd741e 100644 --- a/smop/main.py +++ b/smop/main.py @@ -1,189 +1,81 @@ -# SMOP compiler -- Simple Matlab/Octave to Python compiler -# Copyright 2011-2013 Victor Leikehman +# SMOP -- Simple Matlab/Octave to Python compiler +# Copyright 2011-2016 Victor Leikehman -import version +#import version import sys,cPickle,glob,os import getopt,re import lexer,parse,resolve,backend,options,node,graphviz +import callgraph import networkx as nx +import pickle import readline -#from runtime import * -#from version import __version__ -__version__ = version.__version__ - -def usage(): - print "SMOP compiler version " + __version__ - print """Usage: smop [options] file-list - Options: - -V --version - -X --exclude=FILES Ignore files listed in comma-separated list FILES. - Can be used several times. - -S --syntax-errors=FILES Ignore syntax errors in comma-separated list of FILES. - Can be used several times. - -S. Always gnore syntax errors - -d --dot=REGEX For functions whose names match REGEX, save debugging - information in "dot" format (see www.graphviz.org). - You need an installation of graphviz to use --dot - option. Use "dot" utility to create a pdf file. - For example: - $ python main.py fastsolver.m -d "solver|cbest" - $ dot -Tpdf -o resolve_solver.pdf resolve_solver.dot - -h --help - -o --output=FILENAME By default create file named a.py - -o- --output=- Use standard output - -s --strict Stop on the first error - -v --verbose -""" +import graphviz def main(): - """ - !a="def f(): \\n\\treturn 123" - !exec a - !print f - !print f() - !reload(backend) - =>> function t=foo(a) \\ - ... t=123 - !exec foo(3) - """ - try: - opts, args = getopt.gnu_getopt(sys.argv[1:], - "d:ho:vVsr:S:X:", - [ - "dot=", - "exclude=", - "help", - "syntax-errors=", - "output=", - "runtime=", - "strict", - "verbose", - "version", - ]) - except getopt.GetoptError, err: - # print help information and exit: - print str(err) # will print something like "option -a not recognized" - usage() - sys.exit(2) - - exclude_list = [] - output = None - strict = 0 - dot = None - runtime = [] - - for o, a in opts: - if o in ("-r", "--runtime"): - runtime += [a] - elif o in ("-s", "--strict"): - strict = 1 - elif o in ("-S", "--syntax-errors"): - options.syntax_errors += a.split(",") - elif o in ("-d", "--dot"): - dot = re.compile(a) - elif o in ("-X", "--exclude"): - exclude_list += [ "%s.m" % b for b in a.split(",")] - elif o in ("-v", "--verbose"): - options.verbose += 1 - elif o in ("-V", "--version"): - print "SMOP compiler version " + __version__ - sys.exit() - elif o in ("-h", "--help"): - usage() - sys.exit() - elif o in ("-o", "--output"): - output = a - else: - assert False, "unhandled option" - - """ - if not args: - usage() - sys.exit() - """ - if not args: - symtab = {} - print "? for help" - while 1: - try: - buf = raw_input("octave: ") - if not buf: - continue - while buf[-1] == "\\": - buf = buf[:-1] + "\n" + raw_input("... ") - if buf[0] == '?': - print main.__doc__ - continue - if buf[0] == "!": - try: - exec buf[1:] - except Exception as ex: - print ex - continue - t = parse.parse(buf if buf[-1]=='\n' else buf+'\n') - if not t: - continue - print "t=", repr(t) - print 60*"-" - resolve.resolve(t,symtab) - print "t=", repr(t) - print 60*"-" - print "symtab:",symtab - s = backend.backend(t) - print "python:",s.strip() - try: - print eval(s) - except SyntaxError: - exec s - except EOFError: - return - except Exception as ex: - print ex + #args = options.parser.parse_args() + #for key in dir(args): + # if not key.startswith("_"): + # setattr(options,key,getattr(args,key)) + if not options.filelist: + options.parser.print_help() + return + #assert options.filelist + #xfiles = options.exclude.split(",") if options.exclude else [] + if not options.output: + options.output = "a.py" + fp = open(options.output,"w") if options.output != "-" else sys.stdout - if not output: - output = "a.py" - fp = open(output,"w") if output != "-" else sys.stdout - print >> fp, "# Autogenerated with SMOP version " + __version__ - print >> fp, "# " + " ".join(sys.argv) + print >> fp, "# Autogenerated with SMOP "# + options.version + # for key in dir(options.args): + # if not key.startswith("_"): + # value = getattr(options,key) + # print >> fp, '# %s=%s' % (key,value) print >> fp, "from __future__ import division" - for a in runtime: - print >> fp, "from %s import *" % a + print >> fp, "from core import *" - for pattern in args: - for filename in glob.glob(os.path.expanduser(pattern)): - if not filename.endswith(".m"): - print "\tIngored file: '%s'" % filename + #if options.callgraph: + # C = nx.DiGraph() + for options.filename in options.filelist: + try: + #for options.filename in glob.glob(os.path.expanduser(pattern)): + print >> fp, "#", options.filename + if not options.filename.endswith((".m",".tst")): + print "\tIgnored file: '%s'" % options.filename continue - if os.path.basename(filename) in exclude_list: - print "\tExcluded file: '%s'" % filename + if os.path.basename(options.filename) in options.xfiles: + print "\tExcluded file: '%s'" % options.filename continue if options.verbose: - print filename - buf = open(filename).read().replace("\r\n","\n") - func_list = parse.parse(buf if buf[-1]=='\n' else buf+'\n',filename) - if not func_list and strict: - sys.exit(-1) - - for func_obj in func_list: - try: - func_name = func_obj.head.ident.name - if options.verbose: - print "\t",func_name - except AttributeError: - if options.verbose: - print "\tJunk ignored" - if strict: - sys.exit(-1) - continue - fp0 = open("parse_"+func_name+".dot","w") if dot and dot.match(func_name) else None - if fp0: - graphviz.graphviz(func_obj,fp0) - if options.do_resolve: - G = resolve.resolve(func_obj) - - for func_obj in func_list: - s = backend.backend(func_obj) + print options.filename + buf = open(options.filename).read().replace("\r\n","\n") + stmt_list=parse.parse(buf if buf[-1]=='\n' else buf+'\n') + #assert None not in stmt_list + if not stmt_list and options.strict: + return + if options.enumerate: + for i,stmt_obj in enumerate(stmt_list): + #stmt_class = stmt_obj.__class__.__name__ + print i, stmt_obj + # if i == options.debug_index: + # import pdb ; pdb.set_trace() + # if stmt_class == "func_stmt": + # func_name = stmt_obj.ident.name + # if options.verbose: + # print "\t",func_name + # else: + # func_name = "" + if not options.no_resolve: + G = resolve.resolve(stmt_list) + if not options.no_backend: + s = backend.backend(stmt_list) print >> fp, s + except Exception as e: + print e + if options.strict: + raise + # if options.callgraph: + # pickle.dump(C, open("callgraph.pickle","w")) + #nx.write_dot(C,"callgraph.dot") if __name__ == "__main__": main() diff --git a/smop/node.py b/smop/node.py index aadf2959..591cc7a8 100644 --- a/smop/node.py +++ b/smop/node.py @@ -39,6 +39,7 @@ def extend(cls): return lambda f: (setattr(cls,f.__name__,f) or f) def exceptions(f): + return f def wrapper(self,*args,**kwargs): try: return f(self,*args,**kwargs) @@ -46,6 +47,7 @@ def wrapper(self,*args,**kwargs): print "%s.%s()" % (self.__class__.__name__, f.__name__) raise wrapper.__name__ = f.__name__ + wrapper.__doc__ = f.__doc__ return wrapper class node(object): @@ -100,6 +102,9 @@ def __str__(self): return "\n".join([str(t) for t in self]) def __repr__(self): return "stmt_list(%s)" % list.__repr__(self) + def append(self,s): + assert isinstance(s,node), s.__class__ + list.append(self,s) ##################### # @@ -152,10 +157,18 @@ class let(stmt,recordtype("let", def __str__(self): return "%s=%s" % (str(self.ret), str(self.args)) -class func_decl(stmt,recordtype("func_decl","ident ret args decl_list use_nargin",default=None)): +class func_stmt(stmt,recordtype("func_stmt", + """ + ident + ret + args + stmt_list + use_nargin + """, + default=None)): pass -class lambda_expr(func_decl): +class lambda_expr(func_stmt): pass class function(stmt,recordtype("function","head body")): @@ -178,10 +191,18 @@ class global_stmt(stmt,recordtype("global_stmt","global_list")): def __str__(self): return "global %s" % str(self.global_list) +class persistent_stmt(stmt,recordtype("persistent_stmt","global_list")): + def __str__(self): + return "global %s" % str(self.global_list) + class return_stmt(stmt,namedtuple("return_stmt","ret")): def __str__(self): return "return" +class comment_stmt(stmt,namedtuple("comment_stmt","value")): + def __str__(self): + return "comment" + class end_stmt(stmt,namedtuple("end_stmt","dummy")): def __str__(self): return "end" @@ -194,6 +215,10 @@ class break_stmt(stmt,namedtuple("break_stmt","dummy")): def __str__(self): return "break" +class null_stmt(stmt,namedtuple("null_stmt","")): + def __str__(self): + return ";" + class expr_stmt(stmt,node,recordtype("expr_stmt","expr")): def __str__(self): return str(self.expr) @@ -281,21 +306,21 @@ def __str__(self): # names in caps correspond to fortran funcs builtins_list = [ - "ABS", - "ALL", - "ANY", - "CEILING", - "FIND", - "ISNAN", - "MAXVAL", - "MINVAL", - "MODULO", - "RAND", - "RESHAPE", - "SHAPE", - "SIGN", - "SIZE", - "SUM", +# "ABS", +# "ALL", +# "ANY", +# "CEILING", +# "FIND", +# "ISNAN", +# "MAXVAL", +# "MINVAL", +# "MODULO", +# "RAND", +# "RESHAPE", +# "SHAPE", +# "SIGN", +# "SIZE", +# "SUM", #"abs", "add", # synthetic opcode diff --git a/smop/options.py b/smop/options.py index 672e9ac6..aae26f23 100644 --- a/smop/options.py +++ b/smop/options.py @@ -1,34 +1,86 @@ -subscripts = "square" # "round" +import sys +import argparse,textwrap +from textwrap import dedent +from version import __version__ -"""Row vectors in Matlab can be expressed in Fortran as - either one or two dimensional arrays""" -row_vector_ndim = 1 # 2 +parser = argparse.ArgumentParser( + "smop", + usage="""python main.py [options][file.m ...file.m]""", + description= """SMOP is Small Matlab and Octave to Python compiler""", + epilog= + """ + $ python [-mpdb] lexer.py fastsolver.m [options] + $ python [-mpdb] main.py fastsolver.m [options] + emacs (pdb): pdb main.py fastsolver.m [options] + """, + formatter_class=argparse.RawTextHelpFormatter,) -"""Given Matlab code such as [a b c]=size(X) we heuristically - decide that X is 3-dimensional.""" -rank_guess_from_size = True -"""Given false(1,n), deduce that n is a scalar""" -rank_backward_propagate=1 +#parser.add_argument("--callgraph", metavar="F", type=str) -"""0=not even constants - 1=char constants supported -""" -has_char_constants = 0 +parser.add_argument("--enumerate", action="store_true", +help="""enumerate statements (see --debug-index)""") -do_allocate = 0 -do_resolve = 1 -do_rewrite = 0 -do_rename = 0 # SSA -do_typeof = 0 -do_listing = 0 +parser.add_argument("--debug-index", type=int, +help="""enter pdb when stmt index equals --debug-index""") -debug = False +parser.add_argument("--debug-lexer", action="store_true", +help="enable built-in debugging tools") -uppercase=True # active only with f90 +parser.add_argument("--debug-parser", action="store_true", +help="enable built-in debugging tools") -line_numbering=True #True # uses either # or ! or % +parser.add_argument("filelist", nargs="*", metavar="file.m", type=str) -syntax_errors = [] -verbose = 0 +#parser.add_argument("--graphviz", action="store_true") + +parser.add_argument("--keep-comments", action="store_true", +help="""preserve multiline comments""") + +parser.add_argument("--line-numbers", action="store_true", +help="""include line-numbering information """) + +parser.add_argument("-o", "--output", metavar="file.py", type=str, +help= """default output file name is 'a.py'""") + +parser.add_argument("--no-backend", action="store_true", +help="omit code generation") + +parser.add_argument("--no-resolve", action="store_true", +help="omit name resolution") + +parser.add_argument("-s", "--strict", action="store_true", +help="stop after first syntax error") + +parser.add_argument("--testing-mode",action="store_true", +help= """support special "testing" percent-bang +comments used to write Octave test suite. When +disabled, behave like regular comments.""") + +parser.add_argument("-V", '--version', action='version', + version=__version__) + +parser.add_argument("-v", "--verbose",action="store_true") + +parser.add_argument("-x", "--exclude", metavar="f,g,h", type=str, +help="""comma-separated list of files to ignore""") + +args = parser.parse_args(namespace=sys.modules[__name__]) + +xfiles = args.exclude.split(",") if args.exclude else [] filename = "" + +def foo(): + """ + >>> args = parser.parse_args("a b c".split()) + >>> print args.filelist + ['a', 'b', 'c' + + >>> args = parser.parse_args(["--keep-comments"]) + >>> print args.keep_comments + True + + """ +if __name__ == "__main__": + import doctest + doctest.testmod() diff --git a/smop/parse.py b/smop/parse.py index 3f9b4e14..416a9393 100644 --- a/smop/parse.py +++ b/smop/parse.py @@ -6,16 +6,17 @@ import os import operator import sys +import readline import re -import yacc - -from lexer import tokens +import ply.yacc as yacc +import ply.lex as lex import lexer +from lexer import tokens #import builtins import node -#from node import * +from node import exceptions import resolve,options # ident properties (set in parse.py) @@ -58,68 +59,19 @@ class syntax_error(error): def p_top(p): """ top : - | stmt_list - | top func_decl stmt_list_opt - | top func_decl END_STMT semi_opt - | top func_decl stmt_list END_STMT semi_opt + | top stmt + | top END_STMT """ if len(p) == 1: p[0] = node.stmt_list() - elif len(p) == 2: - p[0] = p[1] else: - # we backpatch the func_decl node - assert p[2].__class__ is node.func_decl - p[2].use_nargin = use_nargin - p[2].use_varargin = use_varargin - - try: - if p[3][-1].__class__ is not node.return_stmt: - p[3].append(node.return_stmt(ret_expr)) - except: - raise syntax_error(p) - p[0] = p[1] - p[0].append(node.function(head=p[2],body=p[3])) - assert isinstance(p[0],node.stmt_list) - -def p_semi_opt(p): - """ - semi_opt : - | semi_opt SEMI - | semi_opt COMMA - """ - pass - -def p_stmt(p): - """ - stmt : continue_stmt - | break_stmt - | expr_stmt - | global_stmt - | persistent_stmt - | command - | for_stmt - | if_stmt - | null_stmt - | return_stmt - | switch_stmt - | try_catch - | while_stmt - | foo_stmt - | unwind - """ - # END_STMT is intentionally left out - p[0] = p[1] - -def p_unwind(p): - """ - unwind : UNWIND_PROTECT stmt_list UNWIND_PROTECT_CLEANUP stmt_list END_UNWIND_PROTECT - """ - p[0] = node.try_catch(try_stmt=p[2], - catch_stmt=node.expr_list(), - finally_stmt=p[4]) + if type(p[2]) != str: + p[0].append(p[2]) +########################################## + +@exceptions def p_arg1(p): """ arg1 : STRING @@ -132,101 +84,65 @@ def p_arg1(p): lineno=p.lineno(1), lexpos=p.lexpos(1)) -def p_args(p): + +@exceptions +def p_arg_list(p): """ - args : arg1 - | args arg1 + arg_list : ident + | arg_list COMMA ident """ if len(p) == 2: + p[1].__class__ = node.param p[0] = node.expr_list([p[1]]) - else: + elif len(p) == 4: p[0] = p[1] - p[0].append(p[2]) + p[3].__class__ = node.param + p[0].append(p[3]) + else: + assert 0 + assert isinstance(p[0],node.expr_list) + for ident in p[0]: + ident.props="A" -def p_command(p): - """ - command : ident args SEMI + +@exceptions +def p_args(p): """ -# if p[1].name == "load": -# # "load filename x" ==> "x=load(filename)" -# # "load filename x y z" ==> "(x,y,z)=load(filename)" -# ret=node.expr_list([node.ident(t.value) for t in p[2][1:]]) -# p[0] = node.funcall(func_expr=p[1], -# args=node.expr_list(p[2]), -# ret=ret) -# else: - p[0] = node.funcall(p[1],p[2]) - -#################### -def p_global_list(p): - """global_list : ident - | global_list ident + args : arg1 + | args arg1 """ if len(p) == 2: - p[0] = node.global_list([p[1]]) - elif len(p) == 3: + p[0] = node.expr_list([p[1]]) + else: p[0] = p[1] p[0].append(p[2]) -def p_global_stmt(p): - """ - global_stmt : GLOBAL global_list SEMI - | GLOBAL ident EQ expr SEMI - """ - p[0] = node.global_stmt(p[2]) - for ident in p[0]: - ident.props="G" # G=global - -def p_foo_stmt(p): - "foo_stmt : expr OROR expr SEMI" - expr1 = p[1][1][0] - expr2 = p[3][1][0] - ident = expr1.ret - args1 = expr1.args - args2 = expr2.args - p[0] = node.let(ret=ident, - args=node.expr("or",node.expr_list([args1,args2]))) - -def p_persistent_stmt(p): + +@exceptions +def p_args_opt(p): """ - persistent_stmt : PERSISTENT global_list SEMI - | PERSISTENT ident EQ expr SEMI + args_opt : + | LPAREN RPAREN + | LPAREN expr_list RPAREN """ -# if len(p) == 4: -# p[0] = node.global_stmt(p[2]) -# for ident in p[0]: -# ident.props="G" # G=global -# else: -# assert p[2].__class__ in (node.let,node.ident), p[2].__class__ -# p[0] = p[2] -# #print p[2] - - -def p_return_stmt(p): - "return_stmt : RETURN SEMI" - p[0] = node.return_stmt(ret=ret_expr) - -def p_continue_stmt(p): - "continue_stmt : CONTINUE SEMI" - p[0] = node.continue_stmt(None) + if len(p) == 1: + p[0] = node.expr_list() + elif len(p) == 3: + p[0] = node.expr_list() + elif len(p) == 4: + assert isinstance(p[2],node.expr_list) + p[0] = p[2] + else: + assert 0 + +@exceptions def p_break_stmt(p): "break_stmt : BREAK SEMI" p[0] = node.break_stmt(None) -# switch-case-otherwise - -def p_switch_stmt(p): - """ - switch_stmt : SWITCH expr semi_opt case_list END_STMT - """ - def backpatch(expr,stmt): - if isinstance(stmt,node.if_stmt): - stmt.cond_expr.args[1] = expr - backpatch(expr,stmt.else_stmt) - backpatch(p[2],p[4]) - p[0] = p[4] - + +@exceptions def p_case_list(p): """ case_list : @@ -248,222 +164,104 @@ def p_case_list(p): else: assert 0 -# try-catch - -def p_try_catch(p): + +@exceptions +def p_cellarray(p): """ - try_catch : TRY stmt_list CATCH stmt_list END_STMT - | TRY stmt_list END_STMT + cellarray : LBRACE RBRACE + | LBRACE expr_list RBRACE + | LBRACE concat_list RBRACE + | LBRACE concat_list SEMI RBRACE """ - assert isinstance(p[2],node.stmt_list) - #assert isinstance(p[4],node.stmt_list) - p[0] = node.try_catch(try_stmt=p[2], - catch_stmt=node.stmt_list(), # FIXME - finally_stmt=node.stmt_list()) + if len(p) == 3: + p[0] = node.cellarray(op="{}",args=node.expr_list()) + else: + p[0] = node.cellarray(op="{}",args=p[2]) + + +@exceptions +def p_cellarrayref(p): + """expr : expr LBRACE expr_list RBRACE + | expr LBRACE RBRACE + """ + args = node.expr_list() if len(p) == 4 else p[3] + assert isinstance(args,node.expr_list) + p[0] = node.cellarrayref(func_expr=p[1],args=args) -def p_null_stmt(p): + +@exceptions +def p_command(p): """ - null_stmt : SEMI - | COMMA + command : ident args SEMI """ - p[0] = None - -def p_func_decl(p): - """func_decl : FUNCTION ident args_opt SEMI - | FUNCTION ret EQ ident args_opt SEMI +# if p[1].name == "load": +# # "load filename x" ==> "x=load(filename)" +# # "load filename x y z" ==> "(x,y,z)=load(filename)" +# ret=node.expr_list([node.ident(t.value) for t in p[2][1:]]) +# p[0] = node.funcall(func_expr=p[1], +# args=node.expr_list(p[2]), +# ret=ret) +# else: + p[0] = node.funcall(p[1],p[2]) + +#################### + +@exceptions +def p_comment_stmt(p): """ - global ret_expr,use_nargin,use_varargin - use_varargin = use_nargin = 0 + comment_stmt : COMMENT + """ + p[0] = node.comment_stmt(p[1]) - if len(p) == 5: - assert isinstance(p[3],node.expr_list) - p[0] = node.func_decl(ident=p[2], - ret=node.expr_list(), - args=p[3]) - ret_expr = node.expr_list() - elif len(p) == 7: - assert isinstance(p[2],node.expr_list) - assert isinstance(p[5],node.expr_list) - p[0] = node.func_decl(ident=p[4], - ret=p[2], - args=p[5]) - ret_expr = p[2] + +@exceptions +def p_concat_list(p): + """ + concat_list : expr_list SEMI expr_list + | concat_list SEMI expr_list + """ + if p[1].__class__ == node.expr_list: + p[0] = node.concat_list([p[1],p[3]]) else: - assert 0 + p[0] = p[1] + p[0].append(p[3]) -def p_args_opt(p): + +@exceptions +def p_continue_stmt(p): + "continue_stmt : CONTINUE SEMI" + p[0] = node.continue_stmt(None) + + +@exceptions +def p_elseif_stmt(p): """ - args_opt : - | LPAREN RPAREN - | LPAREN expr_list RPAREN + elseif_stmt : + | ELSE stmt_list_opt + | ELSEIF expr sep stmt_list_opt elseif_stmt """ if len(p) == 1: - p[0] = node.expr_list() + p[0] = node.stmt_list() elif len(p) == 3: - p[0] = node.expr_list() - elif len(p) == 4: - assert isinstance(p[2],node.expr_list) p[0] = p[2] + elif len(p) == 6: + p[0] = node.if_stmt(cond_expr=p[2], + then_stmt=p[4], + else_stmt=p[5]) else: assert 0 -def p_arg_list(p): + + +@exceptions +def p_error_stmt(p): """ - arg_list : ident - | arg_list COMMA ident + error_stmt : ERROR_STMT SEMI """ - if len(p) == 2: - p[1].__class__ = node.param - p[0] = node.expr_list([p[1]]) - elif len(p) == 4: - p[0] = p[1] - p[3].__class__ = node.param - p[0].append(p[3]) - else: - assert 0 - assert isinstance(p[0],node.expr_list) - for ident in p[0]: - ident.props="A" - -def p_ret(p): - """ - ret : ident - | LBRACKET RBRACKET - | LBRACKET expr_list RBRACKET - """ - if len(p) == 2: - p[0] = node.expr_list([p[1]]) - elif len(p) == 3: - p[0] = node.expr_list([]) - elif len(p) == 4: - assert isinstance(p[2],node.expr_list) - p[0] = p[2] - else: - assert 0 - for ident in p[0]: - ident.props="F" -# end func_decl - -def p_stmt_list_opt(p): - """ - stmt_list_opt : - | stmt_list - """ - if len(p) == 1: - p[0] = node.stmt_list() - else: - p[0] = p[1] - -def p_stmt_list(p): - """ - stmt_list : stmt - | stmt_list stmt - """ - if len(p) == 2: - p[0] = node.stmt_list([p[1]] if p[1] else []) - elif len(p) == 3: - p[0] = p[1] - if p[2]: - p[0].append(p[2]) - else: - assert 0 - -def p_concat_list(p): - """ - concat_list : expr_list SEMI expr_list - | concat_list SEMI expr_list - """ - if p[1].__class__ == node.expr_list: - p[0] = node.concat_list([p[1],p[3]]) - else: - p[0] = p[1] - p[0].append(p[3]) - -def p_expr_list(p): - """ - expr_list : exprs - | exprs COMMA - """ - p[0] = p[1] - -def p_exprs(p): - """ - exprs : expr - | exprs COMMA expr - """ - if len(p) == 2: - p[0] = node.expr_list([p[1]]) - elif len(p) == 4: - p[0] = p[1] - p[0].append(p[3]) - else: - assert(0) - assert isinstance(p[0],node.expr_list) - -def p_expr_stmt(p): - """ - expr_stmt : expr_list SEMI - """ - assert isinstance(p[1],node.expr_list) - p[0] = node.expr_stmt(expr=p[1]) - -def p_while_stmt(p): - """ - while_stmt : WHILE expr SEMI stmt_list END_STMT - """ - assert isinstance(p[4],node.stmt_list) - p[0] = node.while_stmt(cond_expr=p[2], - stmt_list=p[4]) - -def p_separator(p): - """ - sep : COMMA - | SEMI - """ - p[0] = p[1] - -def p_if_stmt(p): - """ - if_stmt : IF expr sep stmt_list_opt elseif_stmt END_STMT - | IF expr error stmt_list_opt elseif_stmt END_STMT - """ - p[0] = node.if_stmt(cond_expr=p[2], - then_stmt=p[4], - else_stmt=p[5]) - -def p_elseif_stmt(p): - """ - elseif_stmt : - | ELSE stmt_list_opt - | ELSEIF expr sep stmt_list_opt elseif_stmt - """ - if len(p) == 1: - p[0] = node.stmt_list() - elif len(p) == 3: - p[0] = p[2] - elif len(p) == 6: - p[0] = node.if_stmt(cond_expr=p[2], - then_stmt=p[4], - else_stmt=p[5]) - else: - assert 0 - - -def p_for_stmt(p): - """ - for_stmt : FOR ident EQ expr SEMI stmt_list END_STMT - | FOR LPAREN ident EQ expr RPAREN SEMI stmt_list END_STMT - | FOR matrix EQ expr SEMI stmt_list END_STMT - """ - if len(p) == 8: - if not isinstance(p[2],node.ident): - raise NotImplementedError("for loop") - p[2].props="I" # I= for-loop iteration variable - p[0] = node.for_stmt(ident=p[2], - expr=p[4], - stmt_list=p[6]) -################ expr ################ + p[0] = node.null_stmt() + +@exceptions def p_expr(p): """expr : ident | end @@ -486,47 +284,8 @@ def p_expr(p): else: p[0] = p[1] -def p_lambda_args(p): - """lambda_args : LPAREN RPAREN - | LPAREN arg_list RPAREN - """ - p[0] = p[2] if len(p) == 4 else node.expr_list() - -def p_lambda_expr(p): - """lambda_expr : HANDLE lambda_args expr - """ - p[0] = node.lambda_expr(args=p[2], ret=p[3]) - -def p_expr_ident(p): - "ident : IDENT" - global use_nargin,use_varargin - if p[1] == "varargin": - use_varargin = 1 - if p[1] == "nargin": - use_nargin = 1 - #import pdb; pdb.set_trace() - p[0] = node.ident(name=p[1], - lineno=p.lineno(1), - lexpos=p.lexpos(1), - column=p.lexpos(1) - p.lexer.lexdata.rfind("\n",0,p.lexpos(1))) - -def p_expr_number(p): - "number : NUMBER" - p[0] = node.number(p[1],lineno=p.lineno(1),lexpos=p.lexpos(1)) - -def p_expr_end(p): - "end : END_EXPR" - p[0] = node.expr(op="end",args=node.expr_list([node.number(0), - node.number(0)])) - -def p_expr_string(p): - "string : STRING" - p[0] = node.string(p[1],lineno=p.lineno(1),lexpos=p.lexpos(1)) - -def p_expr_colon(p): - "colon : COLON" - p[0] = node.expr(op=":",args=node.expr_list()) - + +@exceptions def p_expr1(p): """expr1 : MINUS expr %prec UMINUS | PLUS expr %prec UMINUS @@ -537,76 +296,8 @@ def p_expr1(p): """ p[0] = node.expr(op=p[1],args=node.expr_list([p[2]])) -def p_cellarray(p): - """ - cellarray : LBRACE RBRACE - | LBRACE expr_list RBRACE - | LBRACE concat_list RBRACE - | LBRACE concat_list SEMI RBRACE - """ - if len(p) == 3: - p[0] = node.cellarray(op="{}",args=node.expr_list()) - else: - p[0] = node.cellarray(op="{}",args=p[2]) - -def p_matrix(p): - """matrix : LBRACKET RBRACKET - | LBRACKET concat_list RBRACKET - | LBRACKET concat_list SEMI RBRACKET - | LBRACKET expr_list RBRACKET - | LBRACKET expr_list SEMI RBRACKET - """ - if len(p) == 3: - p[0] = node.matrix() - else: - p[0] = node.matrix(p[2]) - -def p_paren_expr(p): - """ - expr : LPAREN expr RPAREN - """ - p[0] = node.expr(op="parens",args=node.expr_list([p[2]])) - -def p_field_expr(p): - """ - expr : expr FIELD - """ - p[0] = node.expr(op=".", - args=node.expr_list([p[1], - node.ident(name=p[2], - lineno=p.lineno(2), - lexpos=p.lexpos(2))])) - -def p_transpose_expr(p): - # p[2] contains the exact combination of plain and conjugate - # transpose operators, such as "'.''.''''". - "expr : expr TRANSPOSE" - p[0] = node.transpose(p[1],node.string(p[2])) - -def p_cellarrayref(p): - """expr : expr LBRACE expr_list RBRACE - | expr LBRACE RBRACE - """ - args = node.expr_list() if len(p) == 4 else p[3] - assert isinstance(args,node.expr_list) - p[0] = node.cellarrayref(func_expr=p[1],args=args) - -def p_funcall_expr(p): - """expr : expr LPAREN expr_list RPAREN - | expr LPAREN RPAREN - """ - if (0 and len(p)==5 and - len(p[3])==1 and - p[3][0].__class__ is node.expr and - p[3][0].op == ":" and not p[3][0].args): - # foo(:) => ravel(foo) - p[0] = node.funcall(func_expr=node.ident("ravel"), - args=node.expr_list([p[1]])) - else: - args = node.expr_list() if len(p) == 4 else p[3] - assert isinstance(args,node.expr_list) - p[0] = node.funcall(func_expr=p[1],args=args) - + +@exceptions def p_expr2(p): """expr2 : expr AND expr | expr ANDAND expr @@ -696,6 +387,9 @@ def p_expr2(p): p[3].nargout = len(p[1].args[0]) elif p[2] == ".*": p[0] = node.dot(p[1],p[3]) +# elif p[2] == "." and isinstance(p[3],node.expr) and p[3].op=="parens": +# p[0] = node.getfield(p[1],p[3].args[0]) +# raise NotImplementedError(p[3],p.lineno(3),p.lexpos(3)) elif p[2] == ":" and isinstance(p[1],node.expr) and p[1].op==":": # Colon expression means different things depending on the # context. As an array subscript, it is a slice; otherwise, @@ -707,49 +401,456 @@ def p_expr2(p): else: p[0] = node.expr(op=p[2],args=node.expr_list([p[1],p[3]])) -def p_error(p): - #import pdb; pdb.set_trace() - #print "p_error",p - if ("." in options.syntax_errors or - os.path.basename(options.filename) in options.syntax_errors): - return p - raise syntax_error(p) -parser = yacc.yacc(start="top") + +@exceptions +def p_expr_colon(p): + "colon : COLON" + p[0] = node.expr(op=":",args=node.expr_list()) + + +@exceptions +def p_expr_end(p): + "end : END_EXPR" + p[0] = node.expr(op="end",args=node.expr_list([node.number(0), + node.number(0)])) + + +@exceptions +def p_expr_ident(p): + "ident : IDENT" + global use_nargin,use_varargin + if p[1] == "varargin": + use_varargin = 1 + if p[1] == "nargin": + use_nargin = 1 + #import pdb; pdb.set_trace() + p[0] = node.ident(name=p[1], + lineno=p.lineno(1), + lexpos=p.lexpos(1), + column=p.lexpos(1) - p.lexer.lexdata.rfind("\n",0,p.lexpos(1))) + + +@exceptions +def p_expr_list(p): + """ + expr_list : exprs + | exprs COMMA + """ + p[0] = p[1] + + +@exceptions +def p_expr_number(p): + "number : NUMBER" + p[0] = node.number(p[1],lineno=p.lineno(1),lexpos=p.lexpos(1)) + + +@exceptions +def p_expr_stmt(p): + """ + expr_stmt : expr_list SEMI + """ + assert isinstance(p[1],node.expr_list) + p[0] = node.expr_stmt(expr=p[1]) + + +@exceptions +def p_expr_string(p): + "string : STRING" + p[0] = node.string(p[1],lineno=p.lineno(1),lexpos=p.lexpos(1)) + + +@exceptions +def p_exprs(p): + """ + exprs : expr + | exprs COMMA expr + """ + if len(p) == 2: + p[0] = node.expr_list([p[1]]) + elif len(p) == 4: + p[0] = p[1] + p[0].append(p[3]) + else: + assert(0) + assert isinstance(p[0],node.expr_list) + + +@exceptions +def p_field_expr(p): + """ + expr : expr FIELD + """ + p[0] = node.expr(op=".", + args=node.expr_list([p[1], + node.ident(name=p[2], + lineno=p.lineno(2), + lexpos=p.lexpos(2))])) + + +@exceptions +def p_foo_stmt(p): + "foo_stmt : expr OROR expr SEMI" + expr1 = p[1][1][0] + expr2 = p[3][1][0] + ident = expr1.ret + args1 = expr1.args + args2 = expr2.args + p[0] = node.let(ret=ident, + args=node.expr("or",node.expr_list([args1,args2]))) + + +@exceptions +def p_for_stmt(p): + """ + for_stmt : FOR ident EQ expr SEMI stmt_list END_STMT + | FOR LPAREN ident EQ expr RPAREN SEMI stmt_list END_STMT + | FOR matrix EQ expr SEMI stmt_list END_STMT + """ + if len(p) == 8: + if not isinstance(p[2],node.ident): + raise NotImplementedError("for loop") + p[2].props="I" # I= for-loop iteration variable + p[0] = node.for_stmt(ident=p[2], + expr=p[4], + stmt_list=p[6]) +################ expr ################ + + +@exceptions +def p_func_stmt(p): + """func_stmt : FUNCTION ident args_opt SEMI + | FUNCTION ret EQ ident args_opt SEMI + """ + global ret_expr,use_nargin,use_varargin + use_varargin = use_nargin = 0 + + if len(p) == 5: + assert isinstance(p[3],node.expr_list) + p[0] = node.func_stmt(ident=p[2], + ret=node.expr_list(), + args=p[3]) + ret_expr = node.expr_list() + elif len(p) == 7: + assert isinstance(p[2],node.expr_list) + assert isinstance(p[5],node.expr_list) + p[0] = node.func_stmt(ident=p[4], + ret=p[2], + args=p[5]) + ret_expr = p[2] + else: + assert 0 + + +@exceptions +def p_funcall_expr(p): + """expr : expr LPAREN expr_list RPAREN + | expr LPAREN RPAREN + """ + if (0 and len(p)==5 and + len(p[3])==1 and + p[3][0].__class__ is node.expr and + p[3][0].op == ":" and not p[3][0].args): + # foo(:) => ravel(foo) + p[0] = node.funcall(func_expr=node.ident("ravel"), + args=node.expr_list([p[1]])) + else: + args = node.expr_list() if len(p) == 4 else p[3] + assert isinstance(args,node.expr_list) + p[0] = node.funcall(func_expr=p[1],args=args) + + +@exceptions +def p_global_list(p): + """global_list : ident + | global_list ident + """ + if len(p) == 2: + p[0] = node.global_list([p[1]]) + elif len(p) == 3: + p[0] = p[1] + p[0].append(p[2]) + + +@exceptions +def p_global_stmt(p): + """ + global_stmt : GLOBAL global_list SEMI + | GLOBAL ident EQ expr SEMI + """ + p[0] = node.global_stmt(p[2]) + for ident in p[0]: + ident.props="G" # G=global + + +@exceptions +def p_if_stmt(p): + """ + if_stmt : IF expr sep stmt_list_opt elseif_stmt END_STMT + | IF expr error stmt_list_opt elseif_stmt END_STMT + """ + p[0] = node.if_stmt(cond_expr=p[2], + then_stmt=p[4], + else_stmt=p[5]) + + +@exceptions +def p_lambda_args(p): + """lambda_args : LPAREN RPAREN + | LPAREN arg_list RPAREN + """ + p[0] = p[2] if len(p) == 4 else node.expr_list() + + +@exceptions +def p_lambda_expr(p): + """lambda_expr : HANDLE lambda_args expr + """ + p[0] = node.lambda_expr(args=p[2], ret=p[3]) + + +@exceptions +def p_matrix(p): + """matrix : LBRACKET RBRACKET + | LBRACKET concat_list RBRACKET + | LBRACKET concat_list SEMI RBRACKET + | LBRACKET expr_list RBRACKET + | LBRACKET expr_list SEMI RBRACKET + """ + if len(p) == 3: + p[0] = node.matrix() + else: + p[0] = node.matrix(p[2]) + + +@exceptions +def p_null_stmt(p): + """ + null_stmt : SEMI + | COMMA + """ + p[0] = node.null_stmt() + + +@exceptions +def p_paren_expr(p): + """ + expr : LPAREN expr RPAREN + """ + p[0] = node.expr(op="parens",args=node.expr_list([p[2]])) + + +@exceptions +def p_persistent_stmt(p): + """ + persistent_stmt : PERSISTENT global_list SEMI + | PERSISTENT ident EQ expr SEMI + """ + p[0] = node.null_stmt() +# if len(p) == 4: +# p[0] = node.global_stmt(p[2]) +# for ident in p[0]: +# ident.props="G" # G=global +# else: +# assert p[2].__class__ in (node.let,node.ident), p[2].__class__ +# p[0] = p[2] +# #print p[2] + + + +@exceptions +def p_ret(p): + """ + ret : ident + | LBRACKET RBRACKET + | LBRACKET expr_list RBRACKET + """ + if len(p) == 2: + p[0] = node.expr_list([p[1]]) + elif len(p) == 3: + p[0] = node.expr_list([]) + elif len(p) == 4: + assert isinstance(p[2],node.expr_list) + p[0] = p[2] + else: + assert 0 + for ident in p[0]: + ident.props="F" +# end func_decl + + +@exceptions +def p_return_stmt(p): + "return_stmt : RETURN SEMI" + p[0] = node.return_stmt(ret=ret_expr) + + +@exceptions +def p_semi_opt(p): + """ + semi_opt : + | semi_opt SEMI + | semi_opt COMMA + """ + pass + + +@exceptions +def p_separator(p): + """ + sep : COMMA + | SEMI + """ + p[0] = p[1] + + +@exceptions +def p_stmt(p): + """ + stmt : continue_stmt + | comment_stmt + | func_stmt + | break_stmt + | expr_stmt + | global_stmt + | persistent_stmt + | error_stmt + | command + | for_stmt + | if_stmt + | null_stmt + | return_stmt + | switch_stmt + | try_catch + | while_stmt + | foo_stmt + | unwind + """ + # END_STMT is intentionally left out + p[0] = p[1] + #print p[0] + + +@exceptions +def p_stmt_list(p): + """ + stmt_list : stmt + | stmt_list stmt + """ + if len(p) == 2: + p[0] = node.stmt_list([p[1]] if p[1] else []) + elif len(p) == 3: + p[0] = p[1] + if p[2]: + p[0].append(p[2]) + else: + assert 0 + + +@exceptions +def p_stmt_list_opt(p): + """ + stmt_list_opt : + | stmt_list + """ + if len(p) == 1: + p[0] = node.stmt_list() + else: + p[0] = p[1] + + +@exceptions +def p_switch_stmt(p): + """ + switch_stmt : SWITCH expr semi_opt case_list END_STMT + """ + def backpatch(expr,stmt): + if isinstance(stmt,node.if_stmt): + stmt.cond_expr.args[1] = expr + backpatch(expr,stmt.else_stmt) + backpatch(p[2],p[4]) + p[0] = p[4] + + +@exceptions +def p_transpose_expr(p): + # p[2] contains the exact combination of plain and conjugate + # transpose operators, such as "'.''.''''". + "expr : expr TRANSPOSE" + p[0] = node.transpose(p[1],node.string(p[2])) + + +@exceptions +def p_try_catch(p): + """ + try_catch : TRY stmt_list CATCH stmt_list END_STMT + | TRY stmt_list END_STMT + """ + assert isinstance(p[2],node.stmt_list) + #assert isinstance(p[4],node.stmt_list) + p[0] = node.try_catch(try_stmt=p[2], + catch_stmt=node.stmt_list(), # FIXME + finally_stmt=node.stmt_list()) + +@exceptions +def p_unwind(p): + """ + unwind : UNWIND_PROTECT stmt_list UNWIND_PROTECT_CLEANUP stmt_list END_UNWIND_PROTECT + """ + p[0] = node.try_catch(try_stmt=p[2], + catch_stmt=node.expr_list(), + finally_stmt=p[4]) + + +@exceptions +def p_while_stmt(p): + """ + while_stmt : WHILE expr SEMI stmt_list END_STMT + """ + assert isinstance(p[4],node.stmt_list) + p[0] = node.while_stmt(cond_expr=p[2], + stmt_list=p[4]) + +@exceptions +def p_error(p): + raise syntax_error(p) + +parser = yacc.yacc(start="top") + -def parse(buf,filename=""): - options.filename = filename +@exceptions +def parse(buf): try: new_lexer = lexer.new() - p = parser.parse(buf,tracking=1,debug=0,lexer=new_lexer) + p = parser.parse(buf, + tracking=1, + debug=options.debug_parser, + lexer=new_lexer) return p except lexer.IllegalCharacterError as (lineno,column,c): #import pdb; pdb.set_trace() - print 'Error:%s:%s.%s:illegal character:%s' % (filename,lineno,column,c) + print 'Error:%s:%s.%s:illegal character:%s' % (options.filename, + lineno,column,c) return [] except NotImplementedError as e: - print 'Error:%s:not implemented:%s' % (filename,e) + print 'Error:%s:not implemented:%s' % (options.filename,e) return [] except syntax_error as e: try: #import pdb;pdb.set_trace() column=e[0].lexpos - new_lexer.lexdata.rfind("\n",0,e[0].lexpos) - print '%s:%s.%s:syntax error %s=<%s>' % (filename, + print '%s:%s.%s:syntax error %s=<%s>' % (options.filename, e[0].lineno, column, e[0].type, e[0].value) - except: - print "%s:unexpected EOF" % filename + except Exception as ex: + print "Unexpected EOF or other error",ex return [] # def fparse(filename): # buf = open(filename).read() # return parse(buf) -# if p is None: -# if p not in opt_exclude: -# raise syntax_error(p) -# elif p.lexpos not in opt_exclude: -# raise syntax_error(p) # vim: ts=4:sw=4:et:si:ai diff --git a/smop/reord.py b/smop/reord.py new file mode 100644 index 00000000..837279f9 --- /dev/null +++ b/smop/reord.py @@ -0,0 +1,4 @@ +import sys +inp = sys.stdin.read().split("\f") +inp.sort() +sys.stdout.write("\f".join(inp)) diff --git a/smop/resolve.py b/smop/resolve.py index d049478f..dc025c0f 100644 --- a/smop/resolve.py +++ b/smop/resolve.py @@ -132,62 +132,31 @@ def copy_symtab(symtab): new_symtab[k] = copy.copy(v) return new_symtab -@extend(node.function) -def _resolve(self,symtab): - self.head._resolve(symtab) - self.body._resolve(symtab) - self.head.ret._resolve(symtab) - -@extend(node.global_list) -@extend(node.concat_list) -@extend(node.expr_list) -def _resolve(self,symtab): - for expr in self: - expr._resolve(symtab) -@extend(node.global_list) -@extend(node.concat_list) -@extend(node.expr_list) +@extend(node.arrayref) +@extend(node.cellarrayref) +@extend(node.funcall) def _lhs_resolve(self,symtab): - for expr in self: - expr._lhs_resolve(symtab) - -@extend(node.stmt_list) -def _resolve(self,symtab): - for stmt in self: - stmt._resolve(symtab) - -@extend(node.number) -@extend(node.string) -def _resolve(self,symtab): - pass - -# @extend(node.call_stmt) -# def _resolve(self,symtab): -# # TODO: does the order of A and B matter? Only if the -# # evaluation of function args may change the value of the -# # func_expr. -# self.func_expr._resolve(symtab) # A -# self.args._resolve(symtab) # B -# self.ret._lhs_resolve(symtab) - -@extend(node.let) + # Definitely lhs array indexing. It's both a ref and a def. + # Must properly handle cases such as foo(foo(17))=42 + # Does the order of A and B matter? + self.func_expr._resolve(symtab) # A + self.args._resolve(symtab) # B + self.func_expr._lhs_resolve(symtab) + +@extend(node.expr) def _lhs_resolve(self,symtab): - self.args._resolve(symtab) - self.ret._lhs_resolve(symtab) - -@extend(node.let) -def _resolve(self,symtab): - self.args._resolve(symtab) - self.ret._lhs_resolve(symtab) - -@extend(node.func_decl) + if self.op == ".": # see setfield + self.args._resolve(symtab) + self.args[0]._lhs_resolve(symtab) + elif self.op == "[]": + for arg in self.args: + arg._lhs_resolve(symtab) + +@extend(node.expr_stmt) def _resolve(self,symtab): - if self.ident: - self.ident._resolve(symtab) - self.args._lhs_resolve(symtab) - self.ret._resolve(symtab) - + self.expr._resolve(symtab) + @extend(node.for_stmt) def _resolve(self,symtab): symtab_copy = copy_symtab(symtab) @@ -198,7 +167,36 @@ def _resolve(self,symtab): # Handle the case where FOR loop is not executed for k,v in symtab_copy.items(): symtab.setdefault(k,set()).update(v) - + +@extend(node.func_stmt) +def _resolve(self,symtab): + if self.ident: + self.ident._resolve(symtab) + self.args._lhs_resolve(symtab) + self.ret._resolve(symtab) + +@extend(node.global_list) +@extend(node.concat_list) +@extend(node.expr_list) +def _lhs_resolve(self,symtab): + for expr in self: + expr._lhs_resolve(symtab) + +@extend(node.global_list) +@extend(node.concat_list) +@extend(node.expr_list) +def _resolve(self,symtab): + for expr in self: + expr._resolve(symtab) + +@extend(node.global_stmt) +def _resolve(self,symtab): + self.global_list._lhs_resolve(symtab) + +@extend(node.ident) +def _lhs_resolve(self,symtab): + symtab[self.name] = set([self]) + @extend(node.if_stmt) def _resolve(self,symtab): symtab_copy = copy_symtab(symtab) @@ -208,46 +206,34 @@ def _resolve(self,symtab): self.else_stmt._resolve(symtab_copy) for k,v in symtab_copy.items(): symtab.setdefault(k,set()).update(v) - -@extend(node.continue_stmt) # FIXME -@extend(node.break_stmt) # FIXME -def _resolve(self,symtab): - pass - -@extend(node.global_stmt) -def _resolve(self,symtab): - self.global_list._lhs_resolve(symtab) - -@extend(node.return_stmt) + +@extend(node.let) +def _lhs_resolve(self,symtab): + self.args._resolve(symtab) + self.ret._lhs_resolve(symtab) + +@extend(node.let) def _resolve(self,symtab): - self.ret._resolve(symtab) - #symtab.clear() - -@extend(node.expr_stmt) + self.args._resolve(symtab) + self.ret._lhs_resolve(symtab) + +@extend(node.null_stmt) +@extend(node.continue_stmt) +@extend(node.break_stmt) def _resolve(self,symtab): - self.expr._resolve(symtab) - -@extend(node.where_stmt) # FIXME where_stmt ??? -@extend(node.while_stmt) + pass + +@extend(node.setfield) # a subclass of funcall def _resolve(self,symtab): - symtab_copy = copy_symtab(symtab) - self.cond_expr._resolve(symtab) - self.stmt_list._resolve(symtab) - self.cond_expr._resolve(symtab) - self.stmt_list._resolve(symtab) - # Handle the case where WHILE loop is not executed - for k,v in symtab_copy.items(): - symtab.setdefault(k,set()).update(v) - + self.func_expr._resolve(symtab) + self.args._resolve(symtab) + self.args[0]._lhs_resolve(symtab) + @extend(node.try_catch) def _resolve(self,symtab): self.try_stmt._resolve(symtab) self.catch_stmt._resolve(symtab) # ??? - -@extend(node.ident) -def _lhs_resolve(self,symtab): - symtab[self.name] = set([self]) - + @extend(node.ident) def _resolve(self,symtab): if self.defs is None: @@ -257,7 +243,7 @@ def _resolve(self,symtab): except KeyError: # defs == set() means name used, but not defined pass - + @extend(node.arrayref) @extend(node.cellarrayref) @extend(node.funcall) @@ -269,37 +255,51 @@ def _resolve(self,symtab): self.args._resolve(symtab) #if self.ret: # self.ret._lhs_resolve(symtab) - -@extend(node.setfield) # a subclass of funcall -def _resolve(self,symtab): - self.func_expr._resolve(symtab) - self.args._resolve(symtab) - self.args[0]._lhs_resolve(symtab) - -@extend(node.arrayref) -@extend(node.cellarrayref) -@extend(node.funcall) -def _lhs_resolve(self,symtab): - # Definitely lhs array indexing. It's both a ref and a def. - # Must properly handle cases such as foo(foo(17))=42 - # Does the order of A and B matter? - self.func_expr._resolve(symtab) # A - self.args._resolve(symtab) # B - self.func_expr._lhs_resolve(symtab) - + @extend(node.expr) def _resolve(self,symtab): for expr in self.args: expr._resolve(symtab) + +@extend(node.number) +@extend(node.string) +@extend(node.comment_stmt) +def _resolve(self,symtab): + pass -@extend(node.expr) -def _lhs_resolve(self,symtab): - if self.op == ".": # see setfield - self.args._resolve(symtab) - self.args[0]._lhs_resolve(symtab) - elif self.op == "[]": - for arg in self.args: - arg._lhs_resolve(symtab) - +# @extend(node.call_stmt) +# def _resolve(self,symtab): +# # TODO: does the order of A and B matter? Only if the +# # evaluation of function args may change the value of the +# # func_expr. +# self.func_expr._resolve(symtab) # A +# self.args._resolve(symtab) # B +# self.ret._lhs_resolve(symtab) + +@extend(node.return_stmt) +def _resolve(self,symtab): + self.ret._resolve(symtab) + #symtab.clear() + +@extend(node.stmt_list) +def _resolve(self,symtab): + for stmt in self: + stmt._resolve(symtab) + +@extend(node.where_stmt) # FIXME where_stmt ??? +@extend(node.while_stmt) +def _resolve(self,symtab): + symtab_copy = copy_symtab(symtab) + self.cond_expr._resolve(symtab) + self.stmt_list._resolve(symtab) + self.cond_expr._resolve(symtab) + self.stmt_list._resolve(symtab) + # Handle the case where WHILE loop is not executed + for k,v in symtab_copy.items(): + symtab.setdefault(k,set()).update(v) + @extend(node.function) +def _resolve(self,symtab): + self.head._resolve(symtab) + self.body._resolve(symtab) + self.head.ret._resolve(symtab) - diff --git a/smop/test_lexer.py b/smop/test_lexer.py index 2299861f..dc9e1cbd 100644 --- a/smop/test_lexer.py +++ b/smop/test_lexer.py @@ -144,6 +144,7 @@ def test118(self): tok = self.lexer.next() self.assertEqual(tok.value,r'hello\world') + @unittest.skip("FIXME") def test119(self): "Quotes and backslashes in octave strings" self.lexer.input(r'"hello""world"') diff --git a/smop/test_primes.py b/smop/test_primes.py new file mode 100644 index 00000000..3fb29aee --- /dev/null +++ b/smop/test_primes.py @@ -0,0 +1,2 @@ +import octave +octave.primes(1000) diff --git a/smop/test_runtime.py b/smop/test_runtime.py new file mode 100644 index 00000000..6ff4b746 --- /dev/null +++ b/smop/test_runtime.py @@ -0,0 +1,132 @@ +# SMOP compiler runtime support library +# Copyright 2014 Victor Leikehman + +# MIT license +import numpy as np +import unittest +from runtime import * + +class Getitem(unittest.TestCase): + def setUp(self): + self.a = matlabarray(arange_(1,20).reshape(4,5,order="F")) + # 2-dim + def test01(self): + self.assertEqual(self.a[1,1],1) + + def test02(self): + self.assertEqual(self.a[4,5],20) + + def test03(self): + self.assertTrue(isequal_(self.a[:,:],self.a)) + + # 1-dim + def test04(self): + a = matlabarray(arange_(1,20).reshape(4,5,order="F")) + aa = a[:] + bb = matlabarray(arange_(1,20).reshape(-1,1,order="F")) + self.assertTrue(isequal_(aa,bb)) + + def test05(self): + #import pdb; pdb.set_trace() + z = [[11,22,33,44], + [11,22,33,44], + [11,22,33,44], + [11,22,33,44]] + a = matlabarray([11,22,33,44], dtype=int) + self.assertTrue(isequal_(a[ [1,1,1,1] , 1:4], matlabarray(z))) + self.assertTrue(isequal_(a[ [1,1,1,1], : ], matlabarray(z))) + #self.assertTrue(isequal_(a[ [[1,1,1,1]], 1:4], matlabarray([z]))) + + def test06(self): + a=copy_(0) + a[6]=666 + self.assertTrue(isequal_(a, [[0.,0.,0.,0.,0.,666.]])) + +class Expand(unittest.TestCase): + """ + Expand on index error + """ + def setUp(self): + self.a = matlabarray(zeros_(1,4)) + + def test01(self): + self.a[1,5]=1 + self.assertTrue(isequal_(self.a, + matlabarray([[0,0,0,0,1]]))) + + def test02(self): + #with self.assertRaises(IndexError): + a = matlabarray(zeros_(1,4)) + a[5]=1 # single index + self.assertTrue(isequal_(a, + matlabarray([[0,0,0,0,1]]))) + + #@unittest.skip("") + def test03(self): + a=zeros_(1,4) + a[1:10:4]=1 + "[[ 1. 0. 0. 0. 1. 0. 0. 0. 1. 0.]]" + + #@unittest.skip("") + def test04(self): + a=zeros_(1,4) + with self.assertRaises(IndexError): + a[5,5]=1 + b = matlabarray( + [[0,0,0,0,0], + [0,0,0,0,0], + [0,0,0,0,0], + [0,0,0,0,0], + [0,0,0,0,1]]) + self.assertTrue(isequal_(a,b)) + +class Strread(unittest.TestCase): + def test01(self): + a = strread_("0.11 0.22 0.33") + self.assertTrue(isequal_(a,[[0.11,0.22,0.33]])) + + def test02(self): + a,b,c = strread_("0.11 0.22 0.33",nargout=3) + self.assertEqual(a,0.11) + self.assertEqual(b,0.22) + self.assertEqual(c,0.33) + +class Core(unittest.TestCase): + def setUp(self): + self.a = arange_(1,10).reshape(2,5,order="F") + + def test01(self): + b = abs_(-self.a) + self.assertTrue(isequal_(self.a,b)) + + def test02(self): + b = ceil_(self.a) + self.assertTrue(isequal_(self.a,b)) + + def test03(self): + b = false_(2,3) + self.assertTrue(isequal_(size_(b), [[2,3]])) + + def test_zeros(self): + self.assertEqual(zeros_(), 0.0) + self.assertTrue(isequal_(zeros_(2), zeros_(2,2))) + self.assertTrue(isequal_(zeros_(2,2), zeros_(2,2))) + + +#class Copy(unittest.TestCase): +# def setUp(self): +# self.a = zeros_(1,4) +# +# def test01(self): +# b=self.a.copy() +# print self.a +# print b +# self.assertTrue(isequal_(self.a,b)) +# b[1,1]=123 +# self.assertTrue(not isequal_(self.a,b)) +# #c=b.copy() +# #c[1,:]=1 +# #self.assertTrue(not isequal_(b,c)) + +if __name__ == "__main__": + unittest.main() diff --git a/smop/test_solver.py b/smop/test_solver.py new file mode 100644 index 00000000..53e8053e --- /dev/null +++ b/smop/test_solver.py @@ -0,0 +1,34 @@ +import pstats,cProfile +import numpy,time +from solver import * +from core import * + +def main(): + ai = matlabarray(zeros (10,10,dtype=int),dtype=int) + af = copy(ai) + + ai[1,1]=2 + ai[2,2]=3 + ai[3,3]=4 + ai[4,4]=5 + ai[5,5]=1 + + af[9,9]=1 + af[8,8]=2 + af[7,7]=3 + af[6,6]=4 + af[10,10]=5 + + t0 = time.clock() + mv = solver(ai,af,0) + t1 = time.clock() + print t1-t0 + print mv.shape + +if __name__ == '__main__': + main() + """ + cProfile.runctx('main()',globals(),locals(),"Profile.prof") + s = pstats.Stats("Profile.prof") + s.strip_dirs().sort_stats("time").print_stats() + """