From c5a12719194f8a04813b1a5dc62f0fee9fc67733 Mon Sep 17 00:00:00 2001 From: Wordman Date: Tue, 10 Dec 2019 23:48:33 +0200 Subject: [PATCH 1/9] Properly fix `Jump out of too many nested blocks` error (overrides previous fix, as merging them would be too complex and unneeded) --- goto.py | 68 +++++++++++++++++------------ test_goto.py | 120 ++++++++++++++++++++++++++++++--------------------- 2 files changed, 113 insertions(+), 75 deletions(-) diff --git a/goto.py b/goto.py index 3c67459..81a4ae5 100644 --- a/goto.py +++ b/goto.py @@ -86,6 +86,19 @@ def _parse_instructions(code): extended_arg_offset = None yield (dis.opname[opcode], oparg, offset) +def _get_instruction_size(opname, oparg=0): + size = 1 + + extended_arg = oparg >> _BYTECODE.argument_bits + if extended_arg != 0: + size += _get_instruction_size('EXTENDED_ARG', extended_arg) + oparg &= (1 << _BYTECODE.argument_bits) - 1 + + opcode = dis.opmap[opname] + if opcode >= _BYTECODE.have_argument: + size += _BYTECODE.argument.size + + return size def _write_instruction(buf, pos, opname, oparg=0): extended_arg = oparg >> _BYTECODE.argument_bits @@ -164,37 +177,38 @@ def _patch_code(code): target_depth = len(target_stack) if origin_stack[:target_depth] != target_stack: raise SyntaxError('Jump into different block') + + size = 0 + for i in range(len(origin_stack) - target_depth): + size += _get_instruction_size('POP_BLOCK') + size += _get_instruction_size('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit) + + moved_to_end = False + if pos + size > end: + # not enough space, add at end + pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', len(buf) // _BYTECODE.jump_unit) + + if pos > end: + raise SyntaxError('Internal error - not enough bytecode space') + + size += _get_instruction_size('JUMP_ABSOLUTE', end // _BYTECODE.jump_unit) + + moved_to_end = True + pos = len(buf) + buf.extend([0] * size) - failed = False try: for i in range(len(origin_stack) - target_depth): pos = _write_instruction(buf, pos, 'POP_BLOCK') - - if target >= end: - rel_target = (target - pos) // _BYTECODE.jump_unit - oparg_bits = 0 - - while True: - rel_target -= (1 + _BYTECODE.argument.size) // _BYTECODE.jump_unit - if rel_target >> oparg_bits == 0: - pos = _write_instruction(buf, pos, 'EXTENDED_ARG', 0) - break - - oparg_bits += _BYTECODE.argument_bits - if rel_target >> oparg_bits == 0: - break - - pos = _write_instruction(buf, pos, 'JUMP_FORWARD', rel_target) - else: - pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', target // _BYTECODE.jump_unit) - - except (IndexError, struct.error): - failed = True - - if failed or pos > end: - raise SyntaxError('Jump out of too many nested blocks') - - _inject_nop_sled(buf, pos, end) + pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', target // _BYTECODE.jump_unit) + except (IndexError, struct.error) as e: + raise SyntaxError("Internal error", e) + + if moved_to_end: + pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', end // _BYTECODE.jump_unit) + + else: + _inject_nop_sled(buf, pos, end) return _make_code(code, _array_to_bytes(buf)) diff --git a/test_goto.py b/test_goto.py index 7ff1dc1..cabc7b4 100644 --- a/test_goto.py +++ b/test_goto.py @@ -71,63 +71,87 @@ def func(): pytest.raises(SyntaxError, with_goto, func) -if sys.version_info >= (3, 6): - def test_jump_out_of_nested_2_loops(): - @with_goto - def func(): - x = 1 - for i in range(2): - for j in range(2): - # These are more than 256 bytes of bytecode, requiring - # a JUMP_FORWARD below on Python 3.6+, since the absolute - # address would be too large, after leaving two blocks. - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x +def test_jump_out_of_nested_2_loops(): + @with_goto + def func(): + x = 1 + for i in range(2): + for j in range(2): + # These are more than 256 bytes of bytecode + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + + goto .end + label .end + return (i, j) + assert func() == (0, 0) + +def test_jump_out_of_nested_3_loops(): + @with_goto + def func(): + for i in range(2): + for j in range(2): + for k in range(2): goto .end - label .end - return (i, j) + label .end + return (i, j, k) - assert func() == (0, 0) + assert func() == (0, 0, 0) - def test_jump_out_of_nested_3_loops(): - def func(): - for i in range(2): - for j in range(2): - for k in range(2): +def test_jump_out_of_nested_4_loops(): + @with_goto + def func(): + for i in range(2): + for j in range(2): + for k in range(2): + for m in range(2): goto .end - label .end - return (i, j, k) - - pytest.raises(SyntaxError, with_goto, func) -else: - def test_jump_out_of_nested_4_loops(): - @with_goto - def func(): - for i in range(2): - for j in range(2): - for k in range(2): - for m in range(2): - goto .end - label .end - return (i, j, k, m) + label .end + return (i, j, k, m) - assert func() == (0, 0, 0, 0) + assert func() == (0, 0, 0, 0) - def test_jump_out_of_nested_5_loops(): - def func(): - for i in range(2): - for j in range(2): - for k in range(2): - for m in range(2): - for n in range(2): - goto .end - label .end - return (i, j, k, m, n) +def test_jump_out_of_nested_5_loops(): + @with_goto + def func(): + for i in range(2): + for j in range(2): + for k in range(2): + for m in range(2): + for n in range(2): + goto .end + label .end + return (i, j, k, m, n) - pytest.raises(SyntaxError, with_goto, func) + assert func() == (0, 0, 0, 0, 0) + +def test_jump_out_of_nested_11_loops(): + @with_goto + def func(): + x = 1 + for i1 in range(2): + for i2 in range(2): + for i3 in range(2): + for i4 in range(2): + for i5 in range(2): + for i6 in range(2): + for i7 in range(2): + for i8 in range(2): + for i9 in range(2): + for i10 in range(2): + for i11 in range(2): + # These are more than 256 bytes of bytecode + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + + goto .end + label .end + return (i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11) + assert func() == (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) def test_jump_across_loops(): def func(): From 8b431a23da4d52e1291690640fad05a549d4a819 Mon Sep 17 00:00:00 2001 From: condut Date: Wed, 11 Dec 2019 00:46:46 +0200 Subject: [PATCH 2/9] Rename new SyntaxError to be more descriptive in case ever encountered. I was not able to get it to happen for pretty huge functions (much much larger than the ones in the test), though, so there's a good chance it's unreachable. --- goto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/goto.py b/goto.py index 81a4ae5..18fcf1d 100644 --- a/goto.py +++ b/goto.py @@ -189,7 +189,7 @@ def _patch_code(code): pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', len(buf) // _BYTECODE.jump_unit) if pos > end: - raise SyntaxError('Internal error - not enough bytecode space') + raise SyntaxError('Goto in an incredibly huge function') # not sure if reachable size += _get_instruction_size('JUMP_ABSOLUTE', end // _BYTECODE.jump_unit) From c1613f5c2b4d3aabdd20cb12ffb271e71fd7fedd Mon Sep 17 00:00:00 2001 From: condut Date: Sat, 14 Dec 2019 01:27:03 +0200 Subject: [PATCH 3/9] Style & test updates (other updates to commit in following commits) --- goto.py | 25 +++++++++++------------ test_goto.py | 56 +++++++++++----------------------------------------- 2 files changed, 24 insertions(+), 57 deletions(-) diff --git a/goto.py b/goto.py index 18fcf1d..0e79b3e 100644 --- a/goto.py +++ b/goto.py @@ -88,17 +88,17 @@ def _parse_instructions(code): def _get_instruction_size(opname, oparg=0): size = 1 - + extended_arg = oparg >> _BYTECODE.argument_bits if extended_arg != 0: size += _get_instruction_size('EXTENDED_ARG', extended_arg) oparg &= (1 << _BYTECODE.argument_bits) - 1 - + opcode = dis.opmap[opname] - if opcode >= _BYTECODE.have_argument: + if opcode >= _BYTECODE.have_argument: size += _BYTECODE.argument.size - - return size + + return size def _write_instruction(buf, pos, opname, oparg=0): extended_arg = oparg >> _BYTECODE.argument_bits @@ -177,22 +177,22 @@ def _patch_code(code): target_depth = len(target_stack) if origin_stack[:target_depth] != target_stack: raise SyntaxError('Jump into different block') - + size = 0 for i in range(len(origin_stack) - target_depth): size += _get_instruction_size('POP_BLOCK') size += _get_instruction_size('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit) - + moved_to_end = False if pos + size > end: # not enough space, add at end pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', len(buf) // _BYTECODE.jump_unit) - + if pos > end: raise SyntaxError('Goto in an incredibly huge function') # not sure if reachable - + size += _get_instruction_size('JUMP_ABSOLUTE', end // _BYTECODE.jump_unit) - + moved_to_end = True pos = len(buf) buf.extend([0] * size) @@ -202,11 +202,10 @@ def _patch_code(code): pos = _write_instruction(buf, pos, 'POP_BLOCK') pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', target // _BYTECODE.jump_unit) except (IndexError, struct.error) as e: - raise SyntaxError("Internal error", e) - + raise SyntaxError("Internal error") + if moved_to_end: pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', end // _BYTECODE.jump_unit) - else: _inject_nop_sled(buf, pos, end) diff --git a/test_goto.py b/test_goto.py index cabc7b4..6b12f0c 100644 --- a/test_goto.py +++ b/test_goto.py @@ -88,45 +88,6 @@ def func(): assert func() == (0, 0) -def test_jump_out_of_nested_3_loops(): - @with_goto - def func(): - for i in range(2): - for j in range(2): - for k in range(2): - goto .end - label .end - return (i, j, k) - - assert func() == (0, 0, 0) - -def test_jump_out_of_nested_4_loops(): - @with_goto - def func(): - for i in range(2): - for j in range(2): - for k in range(2): - for m in range(2): - goto .end - label .end - return (i, j, k, m) - - assert func() == (0, 0, 0, 0) - -def test_jump_out_of_nested_5_loops(): - @with_goto - def func(): - for i in range(2): - for j in range(2): - for k in range(2): - for m in range(2): - for n in range(2): - goto .end - label .end - return (i, j, k, m, n) - - assert func() == (0, 0, 0, 0, 0) - def test_jump_out_of_nested_11_loops(): @with_goto def func(): @@ -142,11 +103,18 @@ def func(): for i9 in range(2): for i10 in range(2): for i11 in range(2): - # These are more than 256 bytes of bytecode - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - + # These are more than + # 256 bytes of bytecode + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + goto .end label .end return (i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11) From df4038234aa2a679177c2512fcdf708056ecfc07 Mon Sep 17 00:00:00 2001 From: condut Date: Sat, 14 Dec 2019 01:35:36 +0200 Subject: [PATCH 4/9] Refactor the ops sizing/writing to avoid code duplication (This was originally in the second PR, but since you want to take this one first, I think it's a good commit to add here, since it's relevant to this change and would make diffing against future PRs easier) --- goto.py | 53 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/goto.py b/goto.py index 0e79b3e..cc6d746 100644 --- a/goto.py +++ b/goto.py @@ -100,6 +100,15 @@ def _get_instruction_size(opname, oparg=0): return size +def _get_instructions_size(ops): + size = 0 + for op in ops: + if isinstance(op, str): + size += _get_instruction_size(op) + else: + size += _get_instruction_size(*op) + return size + def _write_instruction(buf, pos, opname, oparg=0): extended_arg = oparg >> _BYTECODE.argument_bits if extended_arg != 0: @@ -116,6 +125,13 @@ def _write_instruction(buf, pos, opname, oparg=0): return pos +def _write_instructions(buf, pos, ops): + for op in ops: + if isinstance(op, str): + pos = _write_instruction(buf, pos, op) + else: + pos = _write_instruction(buf, pos, *op) + return pos def _find_labels_and_gotos(code): labels = {} @@ -178,35 +194,28 @@ def _patch_code(code): if origin_stack[:target_depth] != target_stack: raise SyntaxError('Jump into different block') - size = 0 + ops = [] for i in range(len(origin_stack) - target_depth): - size += _get_instruction_size('POP_BLOCK') - size += _get_instruction_size('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit) + ops.append('POP_BLOCK') + ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) - moved_to_end = False - if pos + size > end: - # not enough space, add at end - pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', len(buf) // _BYTECODE.jump_unit) + if pos + _get_instructions_size(ops) > end: + # not enough space, add code at buffer end and jump there + buf_end = len(buf) - if pos > end: - raise SyntaxError('Goto in an incredibly huge function') # not sure if reachable + go_to_end_ops = [('JUMP_ABSOLUTE', buf_end // _BYTECODE.jump_unit)] - size += _get_instruction_size('JUMP_ABSOLUTE', end // _BYTECODE.jump_unit) + if pos + _get_instructions_size(go_to_end_ops) > end: + # not sure if reachable + raise SyntaxError('Goto in an incredibly huge function') - moved_to_end = True - pos = len(buf) - buf.extend([0] * size) + pos = _write_instructions(buf, pos, go_to_end_ops) + _inject_nop_sled(buf, pos, end) - try: - for i in range(len(origin_stack) - target_depth): - pos = _write_instruction(buf, pos, 'POP_BLOCK') - pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', target // _BYTECODE.jump_unit) - except (IndexError, struct.error) as e: - raise SyntaxError("Internal error") - - if moved_to_end: - pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', end // _BYTECODE.jump_unit) + buf.extend([0] * _get_instructions_size(ops)) + _write_instructions(buf, buf_end, ops) else: + pos = _write_instructions(buf, pos, ops) _inject_nop_sled(buf, pos, end) return _make_code(code, _array_to_bytes(buf)) From fedd2cb19f890ebcefae8e1de63fa5931aedb1c5 Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 00:35:35 +0200 Subject: [PATCH 5/9] Added python 3.8 support Added 4 tests, 3 currently disabled (pre-existing issues, especially on pypy). --- goto.py | 48 +++++++++++++++++++++++++++++--------------- test_goto.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 16 deletions(-) diff --git a/goto.py b/goto.py index cc6d746..349e1d9 100644 --- a/goto.py +++ b/goto.py @@ -33,6 +33,8 @@ def __init__(self): self.argument = struct.Struct(' end: diff --git a/test_goto.py b/test_goto.py index 6b12f0c..27c7199 100644 --- a/test_goto.py +++ b/test_goto.py @@ -63,6 +63,18 @@ def func(): assert func() == 0 +def test_jump_out_of_loop_and_survive(): + @with_goto + def func(): + for i in range(10): + for j in range(10): + goto .end + label .end + return (i, j) + + assert func() == (9, 0) + + def test_jump_into_loop(): def func(): for i in range(10): @@ -148,6 +160,22 @@ def func(): assert func() == None +"""def test_jump_out_of_try_block_and_survive(): + @with_goto + def func(): + for i in range(10): + try: + rv = None + goto .end + except: + rv = 'except' + finally: + rv = 'finally' + label .end + return (i, rv) + + assert func() == (9, None)""" + def test_jump_into_try_block(): def func(): try: @@ -159,6 +187,34 @@ def func(): pytest.raises(SyntaxError, with_goto, func) +"""def test_jump_out_of_except_block(): + @with_goto + def func(): + try: + rv = 1 / 0 + except: + rv = 'except' + goto .end + finally: + rv = 'finally' + label .end + return rv + + assert func() == 'except'""" + + +"""def test_jump_into_except_block(): + def func(): + try: + pass + except: + label .block + pass + goto .block + + pytest.raises(SyntaxError, with_goto, func)""" + + def test_jump_to_unknown_label(): def func(): goto .unknown From 5133d8c21e8c394672a612a9154cd2a5e777c842 Mon Sep 17 00:00:00 2001 From: condut Date: Sat, 14 Dec 2019 02:00:11 +0200 Subject: [PATCH 6/9] Updated style & added py38 to tox/travis/readme --- .travis.yml | 2 + README.md | 2 +- goto.py | 506 +++++++++++++++++++++++++-------------------------- test_goto.py | 466 +++++++++++++++++++++++------------------------ tox.ini | 2 +- 5 files changed, 490 insertions(+), 488 deletions(-) diff --git a/.travis.yml b/.travis.yml index c448cc2..77433de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,8 @@ python: - "3.4" - "3.5" - "3.6" + - "3.7" + - "3.8" - "pypy" - "pypy3" diff --git a/README.md b/README.md index 9157dd6..a0c54f7 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Pypi Entry](https://badge.fury.io/py/goto-statement.svg)](https://pypi.python.org/pypi/goto-statement) A function decorator to use `goto` in Python. -Tested on Python 2.6 through 3.6 and PyPy. +Tested on Python 2.6 through 3.8 and PyPy. [![](https://imgs.xkcd.com/comics/goto.png)](https://xkcd.com/292/) diff --git a/goto.py b/goto.py index 349e1d9..80e802c 100644 --- a/goto.py +++ b/goto.py @@ -1,253 +1,253 @@ -import dis -import struct -import array -import types -import functools - - -try: - _array_to_bytes = array.array.tobytes -except AttributeError: - _array_to_bytes = array.array.tostring - - -class _Bytecode: - def __init__(self): - code = (lambda: x if x else y).__code__.co_code - opcode, oparg = struct.unpack_from('BB', code, 2) - - # Starting with Python 3.6, the bytecode format has been changed to use - # 16-bit words (8-bit opcode + 8-bit argument) for each instruction, - # as opposed to previously 24-bit (8-bit opcode + 16-bit argument) for - # instructions that expect an argument or just 8-bit for those that don't. - # https://bugs.python.org/issue26647 - if dis.opname[opcode] == 'POP_JUMP_IF_FALSE': - self.argument = struct.Struct('B') - self.have_argument = 0 - # As of Python 3.6, jump targets are still addressed by their byte - # unit. This, however, is matter to change, so that jump targets, - # in the future, will refer to the code unit (address in bytes / 2). - # https://bugs.python.org/issue26647 - self.jump_unit = 8 // oparg - else: - self.argument = struct.Struct('= _BYTECODE.have_argument: - oparg = extended_arg | _BYTECODE.argument.unpack_from(code, pos)[0] - pos += _BYTECODE.argument.size - - if opcode == dis.EXTENDED_ARG: - extended_arg = oparg << _BYTECODE.argument_bits - extended_arg_offset = offset - continue - - extended_arg = 0 - extended_arg_offset = None - yield (dis.opname[opcode], oparg, offset) - -def _get_instruction_size(opname, oparg=0): - size = 1 - - extended_arg = oparg >> _BYTECODE.argument_bits - if extended_arg != 0: - size += _get_instruction_size('EXTENDED_ARG', extended_arg) - oparg &= (1 << _BYTECODE.argument_bits) - 1 - - opcode = dis.opmap[opname] - if opcode >= _BYTECODE.have_argument: - size += _BYTECODE.argument.size - - return size - -def _get_instructions_size(ops): - size = 0 - for op in ops: - if isinstance(op, str): - size += _get_instruction_size(op) - else: - size += _get_instruction_size(*op) - return size - -def _write_instruction(buf, pos, opname, oparg=0): - extended_arg = oparg >> _BYTECODE.argument_bits - if extended_arg != 0: - pos = _write_instruction(buf, pos, 'EXTENDED_ARG', extended_arg) - oparg &= (1 << _BYTECODE.argument_bits) - 1 - - opcode = dis.opmap[opname] - buf[pos] = opcode - pos += 1 - - if opcode >= _BYTECODE.have_argument: - _BYTECODE.argument.pack_into(buf, pos, oparg) - pos += _BYTECODE.argument.size - - return pos - -def _write_instructions(buf, pos, ops): - for op in ops: - if isinstance(op, str): - pos = _write_instruction(buf, pos, op) - else: - pos = _write_instruction(buf, pos, *op) - return pos - -def _find_labels_and_gotos(code): - labels = {} - gotos = [] - - block_stack = [] - block_counter = 0 - block_exits = [] - - opname1 = oparg1 = offset1 = None - opname2 = oparg2 = offset2 = None - opname3 = oparg3 = offset3 = None - - for opname4, oparg4, offset4 in _parse_instructions(code.co_code): - if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): - if opname2 == 'LOAD_ATTR' and opname3 == 'POP_TOP': - name = code.co_names[oparg1] - if name == 'label': - labels[oparg2] = (offset1, - offset4, - tuple(block_stack)) - elif name == 'goto': - gotos.append((offset1, - offset4, - oparg2, - tuple(block_stack))) - elif opname1 in ('SETUP_LOOP', - 'SETUP_EXCEPT', 'SETUP_FINALLY', - 'SETUP_WITH', 'SETUP_ASYNC_WITH'): - block_counter += 1 - block_stack.append((opname1, block_counter)) - elif not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER': - block_counter += 1 - block_stack.append((opname1, block_counter)) - block_exits.append(offset1 + oparg1) - elif opname1 == 'POP_BLOCK' and block_stack: - block_stack.pop() - elif block_exits and offset1 == block_exits[-1] and block_stack: - block_stack.pop() - block_exits.pop() - - opname1, oparg1, offset1 = opname2, oparg2, offset2 - opname2, oparg2, offset2 = opname3, oparg3, offset3 - opname3, oparg3, offset3 = opname4, oparg4, offset4 - - return labels, gotos - - -def _inject_nop_sled(buf, pos, end): - while pos < end: - pos = _write_instruction(buf, pos, 'NOP') - - -def _patch_code(code): - labels, gotos = _find_labels_and_gotos(code) - buf = array.array('B', code.co_code) - - for pos, end, _ in labels.values(): - _inject_nop_sled(buf, pos, end) - - for pos, end, label, origin_stack in gotos: - try: - _, target, target_stack = labels[label] - except KeyError: - raise SyntaxError('Unknown label {0!r}'.format(code.co_names[label])) - - target_depth = len(target_stack) - if origin_stack[:target_depth] != target_stack: - raise SyntaxError('Jump into different block') - - ops = [] - for block, _ in origin_stack[target_depth:]: - if block == 'FOR_ITER': - ops.append('POP_TOP') - else: - ops.append('POP_BLOCK') - ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) - - if pos + _get_instructions_size(ops) > end: - # not enough space, add code at buffer end and jump there - buf_end = len(buf) - - go_to_end_ops = [('JUMP_ABSOLUTE', buf_end // _BYTECODE.jump_unit)] - - if pos + _get_instructions_size(go_to_end_ops) > end: - # not sure if reachable - raise SyntaxError('Goto in an incredibly huge function') - - pos = _write_instructions(buf, pos, go_to_end_ops) - _inject_nop_sled(buf, pos, end) - - buf.extend([0] * _get_instructions_size(ops)) - _write_instructions(buf, buf_end, ops) - else: - pos = _write_instructions(buf, pos, ops) - _inject_nop_sled(buf, pos, end) - - return _make_code(code, _array_to_bytes(buf)) - - -def with_goto(func_or_code): - if isinstance(func_or_code, types.CodeType): - return _patch_code(func_or_code) - - return functools.update_wrapper( - types.FunctionType( - _patch_code(func_or_code.__code__), - func_or_code.__globals__, - func_or_code.__name__, - func_or_code.__defaults__, - func_or_code.__closure__, - ), - func_or_code - ) +import dis +import struct +import array +import types +import functools + + +try: + _array_to_bytes = array.array.tobytes +except AttributeError: + _array_to_bytes = array.array.tostring + + +class _Bytecode: + def __init__(self): + code = (lambda: x if x else y).__code__.co_code + opcode, oparg = struct.unpack_from('BB', code, 2) + + # Starting with Python 3.6, the bytecode format has been changed to use + # 16-bit words (8-bit opcode + 8-bit argument) for each instruction, + # as opposed to previously 24-bit (8-bit opcode + 16-bit argument) for + # instructions that expect an argument or just 8-bit for those that don't. + # https://bugs.python.org/issue26647 + if dis.opname[opcode] == 'POP_JUMP_IF_FALSE': + self.argument = struct.Struct('B') + self.have_argument = 0 + # As of Python 3.6, jump targets are still addressed by their byte + # unit. This, however, is matter to change, so that jump targets, + # in the future, will refer to the code unit (address in bytes / 2). + # https://bugs.python.org/issue26647 + self.jump_unit = 8 // oparg + else: + self.argument = struct.Struct('= _BYTECODE.have_argument: + oparg = extended_arg | _BYTECODE.argument.unpack_from(code, pos)[0] + pos += _BYTECODE.argument.size + + if opcode == dis.EXTENDED_ARG: + extended_arg = oparg << _BYTECODE.argument_bits + extended_arg_offset = offset + continue + + extended_arg = 0 + extended_arg_offset = None + yield (dis.opname[opcode], oparg, offset) + +def _get_instruction_size(opname, oparg=0): + size = 1 + + extended_arg = oparg >> _BYTECODE.argument_bits + if extended_arg != 0: + size += _get_instruction_size('EXTENDED_ARG', extended_arg) + oparg &= (1 << _BYTECODE.argument_bits) - 1 + + opcode = dis.opmap[opname] + if opcode >= _BYTECODE.have_argument: + size += _BYTECODE.argument.size + + return size + +def _get_instructions_size(ops): + size = 0 + for op in ops: + if isinstance(op, str): + size += _get_instruction_size(op) + else: + size += _get_instruction_size(*op) + return size + +def _write_instruction(buf, pos, opname, oparg=0): + extended_arg = oparg >> _BYTECODE.argument_bits + if extended_arg != 0: + pos = _write_instruction(buf, pos, 'EXTENDED_ARG', extended_arg) + oparg &= (1 << _BYTECODE.argument_bits) - 1 + + opcode = dis.opmap[opname] + buf[pos] = opcode + pos += 1 + + if opcode >= _BYTECODE.have_argument: + _BYTECODE.argument.pack_into(buf, pos, oparg) + pos += _BYTECODE.argument.size + + return pos + +def _write_instructions(buf, pos, ops): + for op in ops: + if isinstance(op, str): + pos = _write_instruction(buf, pos, op) + else: + pos = _write_instruction(buf, pos, *op) + return pos + +def _find_labels_and_gotos(code): + labels = {} + gotos = [] + + block_stack = [] + block_counter = 0 + block_exits = [] + + opname1 = oparg1 = offset1 = None + opname2 = oparg2 = offset2 = None + opname3 = oparg3 = offset3 = None + + for opname4, oparg4, offset4 in _parse_instructions(code.co_code): + if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): + if opname2 == 'LOAD_ATTR' and opname3 == 'POP_TOP': + name = code.co_names[oparg1] + if name == 'label': + labels[oparg2] = (offset1, + offset4, + tuple(block_stack)) + elif name == 'goto': + gotos.append((offset1, + offset4, + oparg2, + tuple(block_stack))) + elif opname1 in ('SETUP_LOOP', + 'SETUP_EXCEPT', 'SETUP_FINALLY', + 'SETUP_WITH', 'SETUP_ASYNC_WITH'): + block_counter += 1 + block_stack.append((opname1, block_counter)) + elif not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER': + block_counter += 1 + block_stack.append((opname1, block_counter)) + block_exits.append(offset1 + oparg1) + elif opname1 == 'POP_BLOCK' and block_stack: + block_stack.pop() + elif block_exits and offset1 == block_exits[-1] and block_stack: + block_stack.pop() + block_exits.pop() + + opname1, oparg1, offset1 = opname2, oparg2, offset2 + opname2, oparg2, offset2 = opname3, oparg3, offset3 + opname3, oparg3, offset3 = opname4, oparg4, offset4 + + return labels, gotos + + +def _inject_nop_sled(buf, pos, end): + while pos < end: + pos = _write_instruction(buf, pos, 'NOP') + + +def _patch_code(code): + labels, gotos = _find_labels_and_gotos(code) + buf = array.array('B', code.co_code) + + for pos, end, _ in labels.values(): + _inject_nop_sled(buf, pos, end) + + for pos, end, label, origin_stack in gotos: + try: + _, target, target_stack = labels[label] + except KeyError: + raise SyntaxError('Unknown label {0!r}'.format(code.co_names[label])) + + target_depth = len(target_stack) + if origin_stack[:target_depth] != target_stack: + raise SyntaxError('Jump into different block') + + ops = [] + for block, _ in origin_stack[target_depth:]: + if block == 'FOR_ITER': + ops.append('POP_TOP') + else: + ops.append('POP_BLOCK') + ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) + + if pos + _get_instructions_size(ops) > end: + # not enough space, add code at buffer end and jump there + buf_end = len(buf) + + go_to_end_ops = [('JUMP_ABSOLUTE', buf_end // _BYTECODE.jump_unit)] + + if pos + _get_instructions_size(go_to_end_ops) > end: + # not sure if reachable + raise SyntaxError('Goto in an incredibly huge function') + + pos = _write_instructions(buf, pos, go_to_end_ops) + _inject_nop_sled(buf, pos, end) + + buf.extend([0] * _get_instructions_size(ops)) + _write_instructions(buf, buf_end, ops) + else: + pos = _write_instructions(buf, pos, ops) + _inject_nop_sled(buf, pos, end) + + return _make_code(code, _array_to_bytes(buf)) + + +def with_goto(func_or_code): + if isinstance(func_or_code, types.CodeType): + return _patch_code(func_or_code) + + return functools.update_wrapper( + types.FunctionType( + _patch_code(func_or_code.__code__), + func_or_code.__globals__, + func_or_code.__name__, + func_or_code.__defaults__, + func_or_code.__closure__, + ), + func_or_code + ) diff --git a/test_goto.py b/test_goto.py index 27c7199..83ebb97 100644 --- a/test_goto.py +++ b/test_goto.py @@ -1,233 +1,233 @@ -import sys -import pytest -from goto import with_goto - -CODE = '''\ -i = 0 -result = [] - -label .start -if i == 10: - goto .end - -result.append(i) -i += 1 -goto .start - -label .end -''' - -EXPECTED = list(range(10)) - - -def test_range_as_code(): - ns = {} - exec(with_goto(compile(CODE, '', 'exec')), ns) - assert ns['result'] == EXPECTED - - -def make_function(code): - lines = ['def func():'] - for line in code: - lines.append(' ' + line) - lines.append(' return result') - - ns = {} - exec('\n'.join(lines), ns) - return ns['func'] - - -def test_range_as_function(): - assert with_goto(make_function(CODE.splitlines()))() == EXPECTED - - -def test_EXTENDED_ARG(): - code = [] - code.append('result = True') - code.append('goto .foo') - for i in range(2**16): - code.append('label .l{0}'.format(i)) - code.append('result = "dead code"') - code.append('label .foo') - assert with_goto(make_function(code))() is True - - -def test_jump_out_of_loop(): - @with_goto - def func(): - for i in range(10): - goto .end - label .end - return i - - assert func() == 0 - - -def test_jump_out_of_loop_and_survive(): - @with_goto - def func(): - for i in range(10): - for j in range(10): - goto .end - label .end - return (i, j) - - assert func() == (9, 0) - - -def test_jump_into_loop(): - def func(): - for i in range(10): - label .loop - goto .loop - - pytest.raises(SyntaxError, with_goto, func) - -def test_jump_out_of_nested_2_loops(): - @with_goto - def func(): - x = 1 - for i in range(2): - for j in range(2): - # These are more than 256 bytes of bytecode - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - - goto .end - label .end - return (i, j) - - assert func() == (0, 0) - -def test_jump_out_of_nested_11_loops(): - @with_goto - def func(): - x = 1 - for i1 in range(2): - for i2 in range(2): - for i3 in range(2): - for i4 in range(2): - for i5 in range(2): - for i6 in range(2): - for i7 in range(2): - for i8 in range(2): - for i9 in range(2): - for i10 in range(2): - for i11 in range(2): - # These are more than - # 256 bytes of bytecode - x += (x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x) - x += (x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x) - x += (x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x) - - goto .end - label .end - return (i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11) - - assert func() == (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) - -def test_jump_across_loops(): - def func(): - for i in range(10): - goto .other_loop - - for i in range(10): - label .other_loop - - pytest.raises(SyntaxError, with_goto, func) - - -def test_jump_out_of_try_block(): - @with_goto - def func(): - try: - rv = None - goto .end - except: - rv = 'except' - finally: - rv = 'finally' - label .end - return rv - - assert func() == None - - -"""def test_jump_out_of_try_block_and_survive(): - @with_goto - def func(): - for i in range(10): - try: - rv = None - goto .end - except: - rv = 'except' - finally: - rv = 'finally' - label .end - return (i, rv) - - assert func() == (9, None)""" - -def test_jump_into_try_block(): - def func(): - try: - label .block - except: - pass - goto .block - - pytest.raises(SyntaxError, with_goto, func) - - -"""def test_jump_out_of_except_block(): - @with_goto - def func(): - try: - rv = 1 / 0 - except: - rv = 'except' - goto .end - finally: - rv = 'finally' - label .end - return rv - - assert func() == 'except'""" - - -"""def test_jump_into_except_block(): - def func(): - try: - pass - except: - label .block - pass - goto .block - - pytest.raises(SyntaxError, with_goto, func)""" - - -def test_jump_to_unknown_label(): - def func(): - goto .unknown - - pytest.raises(SyntaxError, with_goto, func) - - -def test_function_is_copy(): - def func(): - pass - - func.foo = 'bar' - newfunc = with_goto(func) - - assert newfunc is not func - assert newfunc.foo == 'bar' +import sys +import pytest +from goto import with_goto + +CODE = '''\ +i = 0 +result = [] + +label .start +if i == 10: + goto .end + +result.append(i) +i += 1 +goto .start + +label .end +''' + +EXPECTED = list(range(10)) + + +def test_range_as_code(): + ns = {} + exec(with_goto(compile(CODE, '', 'exec')), ns) + assert ns['result'] == EXPECTED + + +def make_function(code): + lines = ['def func():'] + for line in code: + lines.append(' ' + line) + lines.append(' return result') + + ns = {} + exec('\n'.join(lines), ns) + return ns['func'] + + +def test_range_as_function(): + assert with_goto(make_function(CODE.splitlines()))() == EXPECTED + + +def test_EXTENDED_ARG(): + code = [] + code.append('result = True') + code.append('goto .foo') + for i in range(2**16): + code.append('label .l{0}'.format(i)) + code.append('result = "dead code"') + code.append('label .foo') + assert with_goto(make_function(code))() is True + + +def test_jump_out_of_loop(): + @with_goto + def func(): + for i in range(10): + goto .end + label .end + return i + + assert func() == 0 + + +def test_jump_out_of_loop_and_survive(): + @with_goto + def func(): + for i in range(10): + for j in range(10): + goto .end + label .end + return (i, j) + + assert func() == (9, 0) + + +def test_jump_into_loop(): + def func(): + for i in range(10): + label .loop + goto .loop + + pytest.raises(SyntaxError, with_goto, func) + +def test_jump_out_of_nested_2_loops(): + @with_goto + def func(): + x = 1 + for i in range(2): + for j in range(2): + # These are more than 256 bytes of bytecode + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + + goto .end + label .end + return (i, j) + + assert func() == (0, 0) + +def test_jump_out_of_nested_11_loops(): + @with_goto + def func(): + x = 1 + for i1 in range(2): + for i2 in range(2): + for i3 in range(2): + for i4 in range(2): + for i5 in range(2): + for i6 in range(2): + for i7 in range(2): + for i8 in range(2): + for i9 in range(2): + for i10 in range(2): + for i11 in range(2): + # These are more than + # 256 bytes of bytecode + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + + goto .end + label .end + return (i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11) + + assert func() == (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + +def test_jump_across_loops(): + def func(): + for i in range(10): + goto .other_loop + + for i in range(10): + label .other_loop + + pytest.raises(SyntaxError, with_goto, func) + + +def test_jump_out_of_try_block(): + @with_goto + def func(): + try: + rv = None + goto .end + except: + rv = 'except' + finally: + rv = 'finally' + label .end + return rv + + assert func() == None + + +"""def test_jump_out_of_try_block_and_survive(): + @with_goto + def func(): + for i in range(10): + try: + rv = None + goto .end + except: + rv = 'except' + finally: + rv = 'finally' + label .end + return (i, rv) + + assert func() == (9, None)""" + +def test_jump_into_try_block(): + def func(): + try: + label .block + except: + pass + goto .block + + pytest.raises(SyntaxError, with_goto, func) + + +"""def test_jump_out_of_except_block(): + @with_goto + def func(): + try: + rv = 1 / 0 + except: + rv = 'except' + goto .end + finally: + rv = 'finally' + label .end + return rv + + assert func() == 'except'""" + + +"""def test_jump_into_except_block(): + def func(): + try: + pass + except: + label .block + pass + goto .block + + pytest.raises(SyntaxError, with_goto, func)""" + + +def test_jump_to_unknown_label(): + def func(): + goto .unknown + + pytest.raises(SyntaxError, with_goto, func) + + +def test_function_is_copy(): + def func(): + pass + + func.foo = 'bar' + newfunc = with_goto(func) + + assert newfunc is not func + assert newfunc.foo == 'bar' diff --git a/tox.ini b/tox.ini index 67d5756..e06274a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=py26,py27,py34,py35,py36,pypy,pypy3 +envlist=py26,py27,py34,py35,py36,py37,py38,pypy,pypy3 [testenv] deps=pytest From 2d9047c50a4a349d030c9ddf29eeba9302f4b13c Mon Sep 17 00:00:00 2001 From: condut Date: Sat, 14 Dec 2019 02:11:18 +0200 Subject: [PATCH 7/9] fix line endings snafu (you'll squash anyway, so I'm keeping history) --- goto.py | 506 +++++++++++++++++++++++++-------------------------- test_goto.py | 466 +++++++++++++++++++++++------------------------ 2 files changed, 486 insertions(+), 486 deletions(-) diff --git a/goto.py b/goto.py index 80e802c..4603685 100644 --- a/goto.py +++ b/goto.py @@ -1,253 +1,253 @@ -import dis -import struct -import array -import types -import functools - - -try: - _array_to_bytes = array.array.tobytes -except AttributeError: - _array_to_bytes = array.array.tostring - - -class _Bytecode: - def __init__(self): - code = (lambda: x if x else y).__code__.co_code - opcode, oparg = struct.unpack_from('BB', code, 2) - - # Starting with Python 3.6, the bytecode format has been changed to use - # 16-bit words (8-bit opcode + 8-bit argument) for each instruction, - # as opposed to previously 24-bit (8-bit opcode + 16-bit argument) for - # instructions that expect an argument or just 8-bit for those that don't. - # https://bugs.python.org/issue26647 - if dis.opname[opcode] == 'POP_JUMP_IF_FALSE': - self.argument = struct.Struct('B') - self.have_argument = 0 - # As of Python 3.6, jump targets are still addressed by their byte - # unit. This, however, is matter to change, so that jump targets, - # in the future, will refer to the code unit (address in bytes / 2). - # https://bugs.python.org/issue26647 - self.jump_unit = 8 // oparg - else: - self.argument = struct.Struct('= _BYTECODE.have_argument: - oparg = extended_arg | _BYTECODE.argument.unpack_from(code, pos)[0] - pos += _BYTECODE.argument.size - - if opcode == dis.EXTENDED_ARG: - extended_arg = oparg << _BYTECODE.argument_bits - extended_arg_offset = offset - continue - - extended_arg = 0 - extended_arg_offset = None - yield (dis.opname[opcode], oparg, offset) - -def _get_instruction_size(opname, oparg=0): - size = 1 - - extended_arg = oparg >> _BYTECODE.argument_bits - if extended_arg != 0: - size += _get_instruction_size('EXTENDED_ARG', extended_arg) - oparg &= (1 << _BYTECODE.argument_bits) - 1 - - opcode = dis.opmap[opname] - if opcode >= _BYTECODE.have_argument: - size += _BYTECODE.argument.size - - return size - -def _get_instructions_size(ops): - size = 0 - for op in ops: - if isinstance(op, str): - size += _get_instruction_size(op) - else: - size += _get_instruction_size(*op) - return size - -def _write_instruction(buf, pos, opname, oparg=0): - extended_arg = oparg >> _BYTECODE.argument_bits - if extended_arg != 0: - pos = _write_instruction(buf, pos, 'EXTENDED_ARG', extended_arg) - oparg &= (1 << _BYTECODE.argument_bits) - 1 - - opcode = dis.opmap[opname] - buf[pos] = opcode - pos += 1 - - if opcode >= _BYTECODE.have_argument: - _BYTECODE.argument.pack_into(buf, pos, oparg) - pos += _BYTECODE.argument.size - - return pos - -def _write_instructions(buf, pos, ops): - for op in ops: - if isinstance(op, str): - pos = _write_instruction(buf, pos, op) - else: - pos = _write_instruction(buf, pos, *op) - return pos - -def _find_labels_and_gotos(code): - labels = {} - gotos = [] - - block_stack = [] - block_counter = 0 - block_exits = [] - - opname1 = oparg1 = offset1 = None - opname2 = oparg2 = offset2 = None - opname3 = oparg3 = offset3 = None - - for opname4, oparg4, offset4 in _parse_instructions(code.co_code): - if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): - if opname2 == 'LOAD_ATTR' and opname3 == 'POP_TOP': - name = code.co_names[oparg1] - if name == 'label': - labels[oparg2] = (offset1, - offset4, - tuple(block_stack)) - elif name == 'goto': - gotos.append((offset1, - offset4, - oparg2, - tuple(block_stack))) - elif opname1 in ('SETUP_LOOP', - 'SETUP_EXCEPT', 'SETUP_FINALLY', - 'SETUP_WITH', 'SETUP_ASYNC_WITH'): - block_counter += 1 - block_stack.append((opname1, block_counter)) - elif not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER': - block_counter += 1 - block_stack.append((opname1, block_counter)) - block_exits.append(offset1 + oparg1) - elif opname1 == 'POP_BLOCK' and block_stack: - block_stack.pop() - elif block_exits and offset1 == block_exits[-1] and block_stack: - block_stack.pop() - block_exits.pop() - - opname1, oparg1, offset1 = opname2, oparg2, offset2 - opname2, oparg2, offset2 = opname3, oparg3, offset3 - opname3, oparg3, offset3 = opname4, oparg4, offset4 - - return labels, gotos - - -def _inject_nop_sled(buf, pos, end): - while pos < end: - pos = _write_instruction(buf, pos, 'NOP') - - -def _patch_code(code): - labels, gotos = _find_labels_and_gotos(code) - buf = array.array('B', code.co_code) - - for pos, end, _ in labels.values(): - _inject_nop_sled(buf, pos, end) - - for pos, end, label, origin_stack in gotos: - try: - _, target, target_stack = labels[label] - except KeyError: - raise SyntaxError('Unknown label {0!r}'.format(code.co_names[label])) - - target_depth = len(target_stack) - if origin_stack[:target_depth] != target_stack: - raise SyntaxError('Jump into different block') - - ops = [] - for block, _ in origin_stack[target_depth:]: - if block == 'FOR_ITER': - ops.append('POP_TOP') - else: - ops.append('POP_BLOCK') - ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) - - if pos + _get_instructions_size(ops) > end: - # not enough space, add code at buffer end and jump there - buf_end = len(buf) - - go_to_end_ops = [('JUMP_ABSOLUTE', buf_end // _BYTECODE.jump_unit)] - - if pos + _get_instructions_size(go_to_end_ops) > end: - # not sure if reachable - raise SyntaxError('Goto in an incredibly huge function') - - pos = _write_instructions(buf, pos, go_to_end_ops) - _inject_nop_sled(buf, pos, end) - - buf.extend([0] * _get_instructions_size(ops)) - _write_instructions(buf, buf_end, ops) - else: - pos = _write_instructions(buf, pos, ops) - _inject_nop_sled(buf, pos, end) - - return _make_code(code, _array_to_bytes(buf)) - - -def with_goto(func_or_code): - if isinstance(func_or_code, types.CodeType): - return _patch_code(func_or_code) - - return functools.update_wrapper( - types.FunctionType( - _patch_code(func_or_code.__code__), - func_or_code.__globals__, - func_or_code.__name__, - func_or_code.__defaults__, - func_or_code.__closure__, - ), - func_or_code - ) +import dis +import struct +import array +import types +import functools + + +try: + _array_to_bytes = array.array.tobytes +except AttributeError: + _array_to_bytes = array.array.tostring + + +class _Bytecode: + def __init__(self): + code = (lambda: x if x else y).__code__.co_code + opcode, oparg = struct.unpack_from('BB', code, 2) + + # Starting with Python 3.6, the bytecode format has been changed to use + # 16-bit words (8-bit opcode + 8-bit argument) for each instruction, + # as opposed to previously 24-bit (8-bit opcode + 16-bit argument) for + # instructions that expect an argument or just 8-bit for those that don't. + # https://bugs.python.org/issue26647 + if dis.opname[opcode] == 'POP_JUMP_IF_FALSE': + self.argument = struct.Struct('B') + self.have_argument = 0 + # As of Python 3.6, jump targets are still addressed by their byte + # unit. This, however, is matter to change, so that jump targets, + # in the future, will refer to the code unit (address in bytes / 2). + # https://bugs.python.org/issue26647 + self.jump_unit = 8 // oparg + else: + self.argument = struct.Struct('= _BYTECODE.have_argument: + oparg = extended_arg | _BYTECODE.argument.unpack_from(code, pos)[0] + pos += _BYTECODE.argument.size + + if opcode == dis.EXTENDED_ARG: + extended_arg = oparg << _BYTECODE.argument_bits + extended_arg_offset = offset + continue + + extended_arg = 0 + extended_arg_offset = None + yield (dis.opname[opcode], oparg, offset) + +def _get_instruction_size(opname, oparg=0): + size = 1 + + extended_arg = oparg >> _BYTECODE.argument_bits + if extended_arg != 0: + size += _get_instruction_size('EXTENDED_ARG', extended_arg) + oparg &= (1 << _BYTECODE.argument_bits) - 1 + + opcode = dis.opmap[opname] + if opcode >= _BYTECODE.have_argument: + size += _BYTECODE.argument.size + + return size + +def _get_instructions_size(ops): + size = 0 + for op in ops: + if isinstance(op, str): + size += _get_instruction_size(op) + else: + size += _get_instruction_size(*op) + return size + +def _write_instruction(buf, pos, opname, oparg=0): + extended_arg = oparg >> _BYTECODE.argument_bits + if extended_arg != 0: + pos = _write_instruction(buf, pos, 'EXTENDED_ARG', extended_arg) + oparg &= (1 << _BYTECODE.argument_bits) - 1 + + opcode = dis.opmap[opname] + buf[pos] = opcode + pos += 1 + + if opcode >= _BYTECODE.have_argument: + _BYTECODE.argument.pack_into(buf, pos, oparg) + pos += _BYTECODE.argument.size + + return pos + +def _write_instructions(buf, pos, ops): + for op in ops: + if isinstance(op, str): + pos = _write_instruction(buf, pos, op) + else: + pos = _write_instruction(buf, pos, *op) + return pos + +def _find_labels_and_gotos(code): + labels = {} + gotos = [] + + block_stack = [] + block_counter = 0 + block_exits = [] + + opname1 = oparg1 = offset1 = None + opname2 = oparg2 = offset2 = None + opname3 = oparg3 = offset3 = None + + for opname4, oparg4, offset4 in _parse_instructions(code.co_code): + if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): + if opname2 == 'LOAD_ATTR' and opname3 == 'POP_TOP': + name = code.co_names[oparg1] + if name == 'label': + labels[oparg2] = (offset1, + offset4, + tuple(block_stack)) + elif name == 'goto': + gotos.append((offset1, + offset4, + oparg2, + tuple(block_stack))) + elif opname1 in ('SETUP_LOOP', + 'SETUP_EXCEPT', 'SETUP_FINALLY', + 'SETUP_WITH', 'SETUP_ASYNC_WITH'): + block_counter += 1 + block_stack.append((opname1, block_counter)) + elif not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER': + block_counter += 1 + block_stack.append((opname1, block_counter)) + block_exits.append(offset1 + oparg1) + elif opname1 == 'POP_BLOCK' and block_stack: + block_stack.pop() + elif block_exits and offset1 == block_exits[-1] and block_stack: + block_stack.pop() + block_exits.pop() + + opname1, oparg1, offset1 = opname2, oparg2, offset2 + opname2, oparg2, offset2 = opname3, oparg3, offset3 + opname3, oparg3, offset3 = opname4, oparg4, offset4 + + return labels, gotos + + +def _inject_nop_sled(buf, pos, end): + while pos < end: + pos = _write_instruction(buf, pos, 'NOP') + + +def _patch_code(code): + labels, gotos = _find_labels_and_gotos(code) + buf = array.array('B', code.co_code) + + for pos, end, _ in labels.values(): + _inject_nop_sled(buf, pos, end) + + for pos, end, label, origin_stack in gotos: + try: + _, target, target_stack = labels[label] + except KeyError: + raise SyntaxError('Unknown label {0!r}'.format(code.co_names[label])) + + target_depth = len(target_stack) + if origin_stack[:target_depth] != target_stack: + raise SyntaxError('Jump into different block') + + ops = [] + for block, _ in origin_stack[target_depth:]: + if block == 'FOR_ITER': + ops.append('POP_TOP') + else: + ops.append('POP_BLOCK') + ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) + + if pos + _get_instructions_size(ops) > end: + # not enough space, add code at buffer end and jump there + buf_end = len(buf) + + go_to_end_ops = [('JUMP_ABSOLUTE', buf_end // _BYTECODE.jump_unit)] + + if pos + _get_instructions_size(go_to_end_ops) > end: + # not sure if reachable + raise SyntaxError('Goto in an incredibly huge function') + + pos = _write_instructions(buf, pos, go_to_end_ops) + _inject_nop_sled(buf, pos, end) + + buf.extend([0] * _get_instructions_size(ops)) + _write_instructions(buf, buf_end, ops) + else: + pos = _write_instructions(buf, pos, ops) + _inject_nop_sled(buf, pos, end) + + return _make_code(code, _array_to_bytes(buf)) + + +def with_goto(func_or_code): + if isinstance(func_or_code, types.CodeType): + return _patch_code(func_or_code) + + return functools.update_wrapper( + types.FunctionType( + _patch_code(func_or_code.__code__), + func_or_code.__globals__, + func_or_code.__name__, + func_or_code.__defaults__, + func_or_code.__closure__, + ), + func_or_code + ) diff --git a/test_goto.py b/test_goto.py index 83ebb97..27c7199 100644 --- a/test_goto.py +++ b/test_goto.py @@ -1,233 +1,233 @@ -import sys -import pytest -from goto import with_goto - -CODE = '''\ -i = 0 -result = [] - -label .start -if i == 10: - goto .end - -result.append(i) -i += 1 -goto .start - -label .end -''' - -EXPECTED = list(range(10)) - - -def test_range_as_code(): - ns = {} - exec(with_goto(compile(CODE, '', 'exec')), ns) - assert ns['result'] == EXPECTED - - -def make_function(code): - lines = ['def func():'] - for line in code: - lines.append(' ' + line) - lines.append(' return result') - - ns = {} - exec('\n'.join(lines), ns) - return ns['func'] - - -def test_range_as_function(): - assert with_goto(make_function(CODE.splitlines()))() == EXPECTED - - -def test_EXTENDED_ARG(): - code = [] - code.append('result = True') - code.append('goto .foo') - for i in range(2**16): - code.append('label .l{0}'.format(i)) - code.append('result = "dead code"') - code.append('label .foo') - assert with_goto(make_function(code))() is True - - -def test_jump_out_of_loop(): - @with_goto - def func(): - for i in range(10): - goto .end - label .end - return i - - assert func() == 0 - - -def test_jump_out_of_loop_and_survive(): - @with_goto - def func(): - for i in range(10): - for j in range(10): - goto .end - label .end - return (i, j) - - assert func() == (9, 0) - - -def test_jump_into_loop(): - def func(): - for i in range(10): - label .loop - goto .loop - - pytest.raises(SyntaxError, with_goto, func) - -def test_jump_out_of_nested_2_loops(): - @with_goto - def func(): - x = 1 - for i in range(2): - for j in range(2): - # These are more than 256 bytes of bytecode - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - - goto .end - label .end - return (i, j) - - assert func() == (0, 0) - -def test_jump_out_of_nested_11_loops(): - @with_goto - def func(): - x = 1 - for i1 in range(2): - for i2 in range(2): - for i3 in range(2): - for i4 in range(2): - for i5 in range(2): - for i6 in range(2): - for i7 in range(2): - for i8 in range(2): - for i9 in range(2): - for i10 in range(2): - for i11 in range(2): - # These are more than - # 256 bytes of bytecode - x += (x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x) - x += (x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x) - x += (x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x) - - goto .end - label .end - return (i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11) - - assert func() == (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) - -def test_jump_across_loops(): - def func(): - for i in range(10): - goto .other_loop - - for i in range(10): - label .other_loop - - pytest.raises(SyntaxError, with_goto, func) - - -def test_jump_out_of_try_block(): - @with_goto - def func(): - try: - rv = None - goto .end - except: - rv = 'except' - finally: - rv = 'finally' - label .end - return rv - - assert func() == None - - -"""def test_jump_out_of_try_block_and_survive(): - @with_goto - def func(): - for i in range(10): - try: - rv = None - goto .end - except: - rv = 'except' - finally: - rv = 'finally' - label .end - return (i, rv) - - assert func() == (9, None)""" - -def test_jump_into_try_block(): - def func(): - try: - label .block - except: - pass - goto .block - - pytest.raises(SyntaxError, with_goto, func) - - -"""def test_jump_out_of_except_block(): - @with_goto - def func(): - try: - rv = 1 / 0 - except: - rv = 'except' - goto .end - finally: - rv = 'finally' - label .end - return rv - - assert func() == 'except'""" - - -"""def test_jump_into_except_block(): - def func(): - try: - pass - except: - label .block - pass - goto .block - - pytest.raises(SyntaxError, with_goto, func)""" - - -def test_jump_to_unknown_label(): - def func(): - goto .unknown - - pytest.raises(SyntaxError, with_goto, func) - - -def test_function_is_copy(): - def func(): - pass - - func.foo = 'bar' - newfunc = with_goto(func) - - assert newfunc is not func - assert newfunc.foo == 'bar' +import sys +import pytest +from goto import with_goto + +CODE = '''\ +i = 0 +result = [] + +label .start +if i == 10: + goto .end + +result.append(i) +i += 1 +goto .start + +label .end +''' + +EXPECTED = list(range(10)) + + +def test_range_as_code(): + ns = {} + exec(with_goto(compile(CODE, '', 'exec')), ns) + assert ns['result'] == EXPECTED + + +def make_function(code): + lines = ['def func():'] + for line in code: + lines.append(' ' + line) + lines.append(' return result') + + ns = {} + exec('\n'.join(lines), ns) + return ns['func'] + + +def test_range_as_function(): + assert with_goto(make_function(CODE.splitlines()))() == EXPECTED + + +def test_EXTENDED_ARG(): + code = [] + code.append('result = True') + code.append('goto .foo') + for i in range(2**16): + code.append('label .l{0}'.format(i)) + code.append('result = "dead code"') + code.append('label .foo') + assert with_goto(make_function(code))() is True + + +def test_jump_out_of_loop(): + @with_goto + def func(): + for i in range(10): + goto .end + label .end + return i + + assert func() == 0 + + +def test_jump_out_of_loop_and_survive(): + @with_goto + def func(): + for i in range(10): + for j in range(10): + goto .end + label .end + return (i, j) + + assert func() == (9, 0) + + +def test_jump_into_loop(): + def func(): + for i in range(10): + label .loop + goto .loop + + pytest.raises(SyntaxError, with_goto, func) + +def test_jump_out_of_nested_2_loops(): + @with_goto + def func(): + x = 1 + for i in range(2): + for j in range(2): + # These are more than 256 bytes of bytecode + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + + goto .end + label .end + return (i, j) + + assert func() == (0, 0) + +def test_jump_out_of_nested_11_loops(): + @with_goto + def func(): + x = 1 + for i1 in range(2): + for i2 in range(2): + for i3 in range(2): + for i4 in range(2): + for i5 in range(2): + for i6 in range(2): + for i7 in range(2): + for i8 in range(2): + for i9 in range(2): + for i10 in range(2): + for i11 in range(2): + # These are more than + # 256 bytes of bytecode + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + + goto .end + label .end + return (i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11) + + assert func() == (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + +def test_jump_across_loops(): + def func(): + for i in range(10): + goto .other_loop + + for i in range(10): + label .other_loop + + pytest.raises(SyntaxError, with_goto, func) + + +def test_jump_out_of_try_block(): + @with_goto + def func(): + try: + rv = None + goto .end + except: + rv = 'except' + finally: + rv = 'finally' + label .end + return rv + + assert func() == None + + +"""def test_jump_out_of_try_block_and_survive(): + @with_goto + def func(): + for i in range(10): + try: + rv = None + goto .end + except: + rv = 'except' + finally: + rv = 'finally' + label .end + return (i, rv) + + assert func() == (9, None)""" + +def test_jump_into_try_block(): + def func(): + try: + label .block + except: + pass + goto .block + + pytest.raises(SyntaxError, with_goto, func) + + +"""def test_jump_out_of_except_block(): + @with_goto + def func(): + try: + rv = 1 / 0 + except: + rv = 'except' + goto .end + finally: + rv = 'finally' + label .end + return rv + + assert func() == 'except'""" + + +"""def test_jump_into_except_block(): + def func(): + try: + pass + except: + label .block + pass + goto .block + + pytest.raises(SyntaxError, with_goto, func)""" + + +def test_jump_to_unknown_label(): + def func(): + goto .unknown + + pytest.raises(SyntaxError, with_goto, func) + + +def test_function_is_copy(): + def func(): + pass + + func.foo = 'bar' + newfunc = with_goto(func) + + assert newfunc is not func + assert newfunc.foo == 'bar' From 281cc1c1becb754257d551ec79e0abbf88469140 Mon Sep 17 00:00:00 2001 From: condut Date: Sat, 14 Dec 2019 02:16:15 +0200 Subject: [PATCH 8/9] Remove the commented out tests (They'll come back in the next PR). (The new test helps test functionality that needs special attention at least on py38) --- test_goto.py | 45 --------------------------------------------- 1 file changed, 45 deletions(-) diff --git a/test_goto.py b/test_goto.py index 27c7199..41a9f0d 100644 --- a/test_goto.py +++ b/test_goto.py @@ -159,23 +159,6 @@ def func(): assert func() == None - -"""def test_jump_out_of_try_block_and_survive(): - @with_goto - def func(): - for i in range(10): - try: - rv = None - goto .end - except: - rv = 'except' - finally: - rv = 'finally' - label .end - return (i, rv) - - assert func() == (9, None)""" - def test_jump_into_try_block(): def func(): try: @@ -187,34 +170,6 @@ def func(): pytest.raises(SyntaxError, with_goto, func) -"""def test_jump_out_of_except_block(): - @with_goto - def func(): - try: - rv = 1 / 0 - except: - rv = 'except' - goto .end - finally: - rv = 'finally' - label .end - return rv - - assert func() == 'except'""" - - -"""def test_jump_into_except_block(): - def func(): - try: - pass - except: - label .block - pass - goto .block - - pytest.raises(SyntaxError, with_goto, func)""" - - def test_jump_to_unknown_label(): def func(): goto .unknown From b16928d15acde4f1926b9639033521ec3e19ae66 Mon Sep 17 00:00:00 2001 From: condut Date: Sun, 15 Dec 2019 12:58:42 +0200 Subject: [PATCH 9/9] Try reverting premature bugfix that possibly broke a test? --- goto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/goto.py b/goto.py index 1a77f37..d7bed8e 100644 --- a/goto.py +++ b/goto.py @@ -246,7 +246,7 @@ def _patch_code(code): ops = [] - for block, _ in reversed(origin_stack[target_depth:]): + for block, _ in origin_stack[target_depth:]: if block == 'FOR_ITER': ops.append('POP_TOP') else: