feat: Add support for Python 3.12 (#27)

* Python 3.12 changed the meaning of the arg for both COMPARE_OP
      and JUMP instructions:
    
    * For COMPARE_OP, the index now starts at the fifth-lowest bit.
    * For JUMP instructions,
       "The argument of a jump is the offset of the target instruction relative to the instruction that appears immediately after the jump instruction’s CACHE entries.
       As a consequence, the presence of the CACHE instructions is transparent for forward jumps but needs to be taken into account when reasoning about backward jumps."
    
      For both of these, we parse the argrepr. In particular:
    
    * For COMPARE_OP, it be "<", ">", "==", "!=", ">=" or "<=".
    * For JUMP instructions, it be "to BYTECODE_OFFSET", which is exactly
       twice the instruction index.
    
* Python 3.12 changed the meaning of FOR_ITER to leave the iterator on
    the stack (and push an undocumented object) when exhausted,
    and added the instruction END_FOR to pop them both off. However, since
    the target for FOR_ITER is always END_FOR (see https://github.com/python/cpython/blob/17a82a1d16a46b6c008240bcc698619419ce5554/Python/bytecodes.c#L2289-L2293),
   we can "optimize" it by using the old implementation and making END_FOR a no-op.
    
* Python 3.12 merged LOAD_ATTR and LOAD_METHOD into one opcode.
    
* Python 3.12 added BINARY_SLICE and STORE_SLICE opcodes to implement
      `a[start:end]` and `a[start:end] = iterable` (previously,
      BUILD_SLICE was used followed by `GET_ITEM` or `SET_ITEM`).
    
* Python 3.12 added END_SEND for generator cleanup, which we
      implement as a NOOP.
    
* Python 3.12 added CLEANUP_THROW and INTRINSIC_STOPITERATION_ERROR,
      which replace TOS with its value if TOS a StopIteration,
      else reraise the error on TOS. CLEANUP_THROW additionally pops
      two additional values off the stack.
    
* Python 3.12 added LOAD_FAST_CHECK and LOAD_FAST_AND_CLEAR.
      LOAD_FAST now will never raise unbound local error,
      LOAD_FAST_CHECK might raise an unbound local error,
      and LOAD_FAST_AND_CLEAR will load NONE when the local is unbound
      (and additionally sets the local to null).
    
      Since Java rightfully says "???" when it sees code that can access
      an unbound local, we need to set locals used by
      LOAD_FAST_AND_CLEAR to null before generating code
      for opcodes.
    
* Python 3.12 added RETURN_CONST to return a constant.
    
* Python 3.12 changed UNARY_POS from a "fast" opcode to a
      slow "intrinsic" opcode since "+a" is rarely used. This
      does not affect us (other than having a new alternative
      way of writing "UNARY_POS" as
      "INTRINSIC_1_INTRINSIC_UNARY_POSITIVE".
    
* Simplified and fix some bugs in PythonSlice code
    
* Add a workaround for jpype-project/jpype#1178

Author: Christopher-Chianelli

.github/workflows/pull_request.yml

@@ -72,13 +72,13 @@ jobs:
           distribution: 'temurin'
           cache: 'maven'
       # Need to install all Python versions in the same run for tox
-      - name: Python 3.9, Python 3.10, Python 3.11 Setup
+      - name: Python 3.10, Python 3.11, Python 3.12 Setup
         uses: actions/setup-python@v4
         with:
           python-version: |
-              3.9
               3.10
               3.11
+              3.12
           cache: 'pip'
           cache-dependency-path: |
             **/setup.py

.github/workflows/sonarcloud.yml

@@ -77,13 +77,13 @@ jobs:
           key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
           restore-keys: ${{ runner.os }}-m2
           # Need to install all Python versions in the same run for tox
-      - name: Python 3.9, Python 3.10, Python 3.11 Setup
+      - name: Python 3.10, Python 3.11, Python 3.12 Setup
         uses: actions/setup-python@v4
         with:
           python-version: |
-            3.9
             3.10
             3.11
+            3.12
           cache: 'pip'
           cache-dependency-path: |
             **/setup.py

jpyinterpreter/setup.py

@@ -77,9 +77,9 @@ def run(self):
         'org-stubs': 'src/main/resources',
     },
     test_suite='tests',
-    python_requires='>=3.9',
+    python_requires='>=3.10',
     install_requires=[
-        'JPype1>=1.4.1',
+        'JPype1>=1.5.0',
     ],
     cmdclass={'build_py': FetchDependencies},
     package_data={

jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/CompareOp.java

@@ -1,17 +1,19 @@
 package ai.timefold.jpyinterpreter;
 
+import java.util.Objects;
+
 public enum CompareOp {
-    LESS_THAN(0, "__lt__"),
-    LESS_THAN_OR_EQUALS(1, "__le__"),
-    EQUALS(2, "__eq__"),
-    NOT_EQUALS(3, "__ne__"),
-    GREATER_THAN(4, "__gt__"),
-    GREATER_THAN_OR_EQUALS(5, "__ge__");
+    LESS_THAN("<", "__lt__"),
+    LESS_THAN_OR_EQUALS("<=", "__le__"),
+    EQUALS("==", "__eq__"),
+    NOT_EQUALS("!=", "__ne__"),
+    GREATER_THAN(">", "__gt__"),
+    GREATER_THAN_OR_EQUALS(">=", "__ge__");
 
-    public final int id;
+    public final String id;
     public final String dunderMethod;
 
-    CompareOp(int id, String dunderMethod) {
+    CompareOp(String id, String dunderMethod) {
         this.id = id;
         this.dunderMethod = dunderMethod;
     }
@@ -25,9 +27,9 @@ public static CompareOp getOpForDunderMethod(String dunderMethod) {
         throw new IllegalArgumentException("No Op corresponds to dunder method (" + dunderMethod + ")");
     }
 
-    public static CompareOp getOp(int id) {
+    public static CompareOp getOp(String id) {
         for (CompareOp op : CompareOp.values()) {
-            if (op.id == id) {
+            if (Objects.equals(op.id, id)) {
                 return op;
             }
         }

jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/PythonBytecodeInstruction.java

@@ -4,30 +4,36 @@
 
 import ai.timefold.jpyinterpreter.opcodes.descriptor.OpcodeDescriptor;
 
-public record PythonBytecodeInstruction(String opname, int offset, int arg, OptionalInt startsLine,
+public record PythonBytecodeInstruction(String opname, int offset, int arg,
+        String argRepr, OptionalInt startsLine,
         boolean isJumpTarget) {
     public static PythonBytecodeInstruction atOffset(String opname, int offset) {
-        return new PythonBytecodeInstruction(opname, offset, 0, OptionalInt.empty(), false);
+        return new PythonBytecodeInstruction(opname, offset, 0, "", OptionalInt.empty(), false);
     }
 
     public static PythonBytecodeInstruction atOffset(OpcodeDescriptor instruction, int offset) {
         return atOffset(instruction.name(), offset);
     }
 
     public PythonBytecodeInstruction withArg(int newArg) {
-        return new PythonBytecodeInstruction(opname, offset, newArg, startsLine, isJumpTarget);
+        return new PythonBytecodeInstruction(opname, offset, newArg, argRepr, startsLine, isJumpTarget);
+    }
+
+    public PythonBytecodeInstruction withArgRepr(String newArgRepr) {
+        return new PythonBytecodeInstruction(opname, offset, arg, newArgRepr, startsLine, isJumpTarget);
     }
 
     public PythonBytecodeInstruction startsLine(int lineNumber) {
-        return new PythonBytecodeInstruction(opname, offset, arg, OptionalInt.of(lineNumber), isJumpTarget);
+        return new PythonBytecodeInstruction(opname, offset, arg, argRepr, OptionalInt.of(lineNumber),
+                isJumpTarget);
     }
 
     public PythonBytecodeInstruction withIsJumpTarget(boolean isJumpTarget) {
-        return new PythonBytecodeInstruction(opname, offset, arg, startsLine, isJumpTarget);
+        return new PythonBytecodeInstruction(opname, offset, arg, argRepr, startsLine, isJumpTarget);
     }
 
     public PythonBytecodeInstruction markAsJumpTarget() {
-        return new PythonBytecodeInstruction(opname, offset, arg, startsLine, true);
+        return new PythonBytecodeInstruction(opname, offset, arg, argRepr, startsLine, true);
     }
 
     @Override

jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/PythonBytecodeToJavaBytecodeTranslator.java

@@ -8,11 +8,13 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeSet;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
@@ -30,6 +32,7 @@
 import ai.timefold.jpyinterpreter.opcodes.OpcodeWithoutSource;
 import ai.timefold.jpyinterpreter.opcodes.SelfOpcodeWithoutSource;
 import ai.timefold.jpyinterpreter.opcodes.descriptor.GeneratorOpDescriptor;
+import ai.timefold.jpyinterpreter.opcodes.variable.LoadFastAndClearOpcode;
 import ai.timefold.jpyinterpreter.types.BuiltinTypes;
 import ai.timefold.jpyinterpreter.types.PythonLikeFunction;
 import ai.timefold.jpyinterpreter.types.PythonLikeType;
@@ -175,7 +178,7 @@ public static <T> T forceTranslatePythonBytecodeToGenerator(PythonCompiledFuncti
         Method methodWithoutGenerics = getFunctionalInterfaceMethod(javaFunctionalInterfaceType);
         MethodDescriptor methodDescriptor = new MethodDescriptor(javaFunctionalInterfaceType,
                 methodWithoutGenerics,
-                List.of());
+                Collections.emptyList());
         Class<T> compiledClass = forceTranslatePythonBytecodeToGeneratorClass(pythonCompiledFunction, methodDescriptor,
                 methodWithoutGenerics, false);
         return FunctionImplementor.createInstance(new PythonLikeTuple(), new PythonLikeDict(),
@@ -252,7 +255,7 @@ public static <T> Class<T> translatePythonBytecodeToClass(PythonCompiledFunction
                 null);
 
         translatePythonBytecodeToMethod(methodDescriptor, internalClassName, methodVisitor, pythonCompiledFunction,
-                isPythonLikeFunction, Integer.MAX_VALUE, isVirtual); // TODO: Use actual python version
+                isPythonLikeFunction, isVirtual);
 
         classWriter.visitEnd();
 
@@ -296,7 +299,7 @@ public static <T> Class<T> translatePythonBytecodeToClass(PythonCompiledFunction
                 null);
 
         translatePythonBytecodeToMethod(methodDescriptor, internalClassName, methodVisitor, pythonCompiledFunction,
-                isPythonLikeFunction, Integer.MAX_VALUE, isVirtual); // TODO: Use actual python version
+                isPythonLikeFunction, isVirtual);
 
         String withoutGenericsSignature = Type.getMethodDescriptor(methodWithoutGenerics);
         if (!withoutGenericsSignature.equals(methodDescriptor.getMethodDescriptor())) {
@@ -991,7 +994,7 @@ public static PythonFunctionType getFunctionType(PythonCompiledFunction pythonCo
     }
 
     private static void translatePythonBytecodeToMethod(MethodDescriptor method, String className, MethodVisitor methodVisitor,
-            PythonCompiledFunction pythonCompiledFunction, boolean isPythonLikeFunction, int pythonVersion, boolean isVirtual) {
+            PythonCompiledFunction pythonCompiledFunction, boolean isPythonLikeFunction, boolean isVirtual) {
         // Apply Method Adapters, which reorder try blocks and check the bytecode to ensure it valid
         methodVisitor = MethodVisitorAdapters.adapt(methodVisitor, method);
 
@@ -1057,6 +1060,7 @@ private static void translatePythonBytecodeToMethod(MethodDescriptor method, Str
 
         FlowGraph flowGraph = FlowGraph.createFlowGraph(functionMetadata, initialStackMetadata, opcodeList);
         List<StackMetadata> stackMetadataForOpcodeIndex = flowGraph.getStackMetadataForOperations();
+
         writeInstructionsForOpcodes(functionMetadata, stackMetadataForOpcodeIndex, opcodeList);
 
         methodVisitor.visitLabel(end);
@@ -1140,6 +1144,19 @@ public static void writeInstructionsForOpcodes(FunctionMetadata functionMetadata
                     });
         }
 
+        var requiredNullVariableSet = new TreeSet<Integer>();
+        for (Opcode opcode : opcodeList) {
+            if (opcode instanceof LoadFastAndClearOpcode loadAndClearOpcode) {
+                requiredNullVariableSet.add(loadAndClearOpcode.getInstruction().arg());
+            }
+        }
+
+        for (var requiredNullVariable : requiredNullVariableSet) {
+            methodVisitor.visitInsn(Opcodes.ACONST_NULL);
+            methodVisitor.visitVarInsn(Opcodes.ASTORE,
+                    stackMetadataForOpcodeIndex.get(0).localVariableHelper.getPythonLocalVariableSlot(requiredNullVariable));
+        }
+
         for (int i = 0; i < opcodeList.size(); i++) {
             StackMetadata stackMetadata = stackMetadataForOpcodeIndex.get(i);
             PythonBytecodeInstruction instruction = pythonCompiledFunction.instructionList.get(i);
@@ -1148,7 +1165,7 @@ public static void writeInstructionsForOpcodes(FunctionMetadata functionMetadata
                 Label label = exceptionTableTargetLabelMap.get(instruction.offset());
                 methodVisitor.visitLabel(label);
             }
-            exceptionTableTryBlockMap.getOrDefault(instruction.offset(), List.of()).forEach(Runnable::run);
+            exceptionTableTryBlockMap.getOrDefault(instruction.offset(), Collections.emptyList()).forEach(Runnable::run);
 
             if (instruction.isJumpTarget() || bytecodeCounterToLabelMap.containsKey(instruction.offset())) {
                 Label label = bytecodeCounterToLabelMap.computeIfAbsent(instruction.offset(), offset -> new Label());
@@ -1157,7 +1174,7 @@ public static void writeInstructionsForOpcodes(FunctionMetadata functionMetadata
 
             runAfterLabelAndBeforeArgumentors.accept(instruction);
 
-            bytecodeIndexToArgumentorsMap.getOrDefault(instruction.offset(), List.of()).forEach(Runnable::run);
+            bytecodeIndexToArgumentorsMap.getOrDefault(instruction.offset(), Collections.emptyList()).forEach(Runnable::run);
 
             if (exceptionTableStartLabelMap.containsKey(instruction.offset())) {
                 Label label = exceptionTableStartLabelMap.get(instruction.offset());

jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/PythonClassTranslator.java

@@ -26,6 +26,7 @@
 import ai.timefold.jpyinterpreter.opcodes.AbstractOpcode;
 import ai.timefold.jpyinterpreter.opcodes.Opcode;
 import ai.timefold.jpyinterpreter.opcodes.SelfOpcodeWithoutSource;
+import ai.timefold.jpyinterpreter.opcodes.controlflow.ReturnConstantValueOpcode;
 import ai.timefold.jpyinterpreter.opcodes.controlflow.ReturnValueOpcode;
 import ai.timefold.jpyinterpreter.opcodes.object.DeleteAttrOpcode;
 import ai.timefold.jpyinterpreter.opcodes.object.LoadAttrOpcode;
@@ -788,7 +789,7 @@ private static void createInstanceMethod(PythonLikeType pythonLikeType, ClassWri
                         function.getReturnType().orElse(BuiltinTypes.BASE_TYPE),
                         function.totalArgCount() > 0
                                 ? function.getParameterTypes().subList(1, function.getParameterTypes().size())
-                                : List.of()));
+                                : Collections.emptyList()));
     }
 
     private static void createStaticMethod(PythonLikeType pythonLikeType, ClassWriter classWriter, String internalClassName,
@@ -1394,6 +1395,9 @@ public static PythonLikeType getPythonReturnTypeOfFunction(PythonCompiledFunctio
             flowGraph.visitOperations(ReturnValueOpcode.class, (opcode, stackMetadata) -> {
                 possibleReturnTypeList.add(stackMetadata.getTOSType());
             });
+            flowGraph.visitOperations(ReturnConstantValueOpcode.class, (opcode, stackMetadata) -> {
+                possibleReturnTypeList.add(opcode.getConstant(pythonCompiledFunction).$getGenericType());
+            });
 
             return possibleReturnTypeList.stream()
                     .reduce(PythonLikeType::unifyWith)

jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/PythonFunctionSignature.java

@@ -56,12 +56,13 @@ public static PythonFunctionSignature forMethod(Method method) {
 
     public PythonFunctionSignature(MethodDescriptor methodDescriptor,
             PythonLikeType returnType, PythonLikeType... parameterTypes) {
-        this(methodDescriptor, List.of(), extractKeywordArgument(methodDescriptor), returnType, parameterTypes);
+        this(methodDescriptor, Collections.emptyList(), extractKeywordArgument(methodDescriptor), returnType, parameterTypes);
     }
 
     public PythonFunctionSignature(MethodDescriptor methodDescriptor,
             PythonLikeType returnType, List<PythonLikeType> parameterTypeList) {
-        this(methodDescriptor, List.of(), extractKeywordArgument(methodDescriptor), returnType, parameterTypeList);
+        this(methodDescriptor, Collections.emptyList(), extractKeywordArgument(methodDescriptor), returnType,
+                parameterTypeList);
     }
 
     public PythonFunctionSignature(MethodDescriptor methodDescriptor,

jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/PythonVersion.java

@@ -5,11 +5,11 @@
 public final class PythonVersion implements Comparable<PythonVersion> {
     private final int hexversion;
 
-    public static final PythonVersion PYTHON_3_9 = new PythonVersion(3, 9);
     public static final PythonVersion PYTHON_3_10 = new PythonVersion(3, 10);
     public static final PythonVersion PYTHON_3_11 = new PythonVersion(3, 11);
+    public static final PythonVersion PYTHON_3_12 = new PythonVersion(3, 12);
 
-    public static final PythonVersion MINIMUM_PYTHON_VERSION = PYTHON_3_9;
+    public static final PythonVersion MINIMUM_PYTHON_VERSION = PYTHON_3_10;
 
     public PythonVersion(int hexversion) {
         this.hexversion = hexversion;

jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/StackMetadata.java

@@ -2,6 +2,7 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
@@ -41,7 +42,7 @@ public StackMetadata(LocalVariableHelper localVariableHelper) {
             cellVariableValueSources.add(ValueSourceInfo.of(new OpcodeWithoutSource(),
                     BuiltinTypes.BASE_TYPE));
         }
-        this.callKeywordNameList = List.of();
+        this.callKeywordNameList = Collections.emptyList();
     }
 
     private StackMetadata(LocalVariableHelper localVariableHelper, List<ValueSourceInfo> stackValueSources,
@@ -62,11 +63,6 @@ public int getStackSize() {
         return stackValueSources.size();
     }
 
-    public List<PythonLikeType> getStackTypeList() {
-        return stackValueSources.stream().map(ValueSourceInfo::getValueType)
-                .collect(Collectors.toList());
-    }
-
     /**
      * Returns the list index for the given stack index (stack index is how many
      * elements below TOS (i.e. 0 is TOS, 1 is TOS1)).
@@ -113,7 +109,8 @@ public PythonLikeType getTypeAtStackIndex(int index) {
         if (valueSourceInfo != null) {
             return valueSourceInfo.valueType;
         }
-        return null;
+        // Unknown type
+        return BuiltinTypes.BASE_TYPE;
     }
 
     /**
@@ -126,20 +123,6 @@ public ValueSourceInfo getLocalVariableValueSource(int index) {
         return localVariableValueSources.get(index);
     }
 
-    /**
-     * Returns the type for the local variable in slot {@code index}
-     *
-     * @param index The slot
-     * @return The type for the local variable in the given slot
-     */
-    public PythonLikeType getLocalVariableType(int index) {
-        ValueSourceInfo valueSourceInfo = localVariableValueSources.get(index);
-        if (valueSourceInfo != null) {
-            return valueSourceInfo.valueType;
-        }
-        return null;
-    }
-
     /**
      * Returns the value source for the cell variable in slot {@code index}
      *
@@ -150,20 +133,6 @@ public ValueSourceInfo getCellVariableValueSource(int index) {
         return cellVariableValueSources.get(index);
     }
 
-    /**
-     * Returns the type for the cell variable in slot {@code index}
-     *
-     * @param index The slot
-     * @return The type for the cell variable in the given slot
-     */
-    public PythonLikeType getCellVariableType(int index) {
-        ValueSourceInfo valueSourceInfo = cellVariableValueSources.get(index);
-        if (valueSourceInfo != null) {
-            return valueSourceInfo.valueType;
-        }
-        return null;
-    }
-
     public PythonLikeType getTOSType() {
         return getTypeAtStackIndex(0);
     }
@@ -277,13 +246,6 @@ public StackMetadata pushTemps(PythonLikeType... types) {
         return out;
     }
 
-    public StackMetadata insertTemp(int tosIndex, PythonLikeType type) {
-        StackMetadata out = copy();
-        out.stackValueSources.add(stackValueSources.size() - tosIndex,
-                ValueSourceInfo.of(new OpcodeWithoutSource(), type));
-        return out;
-    }
-
     /**
      * Return a new StackMetadata with {@code types} as the stack;
      * The original stack is cleared.