From 8907bc29ed41f46a747aca493b4331d7747bf589 Mon Sep 17 00:00:00 2001 From: Lilith Hafner Date: Fri, 22 Sep 2023 15:37:48 -0500 Subject: [PATCH 01/18] add PyCall extension --- Project.toml | 5 ++++- ext/PyCallExt.jl | 20 ++++++++++++++++++++ test/downstream/pycall.jl | 39 +++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 3 +++ 4 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 ext/PyCallExt.jl create mode 100644 test/downstream/pycall.jl diff --git a/Project.toml b/Project.toml index 364102ac5..04c76aba6 100644 --- a/Project.toml +++ b/Project.toml @@ -33,9 +33,11 @@ TruncatedStacktraces = "781d530d-4396-4725-bb49-402e4bee1e77" ZygoteRules = "700de1a5-db45-46bc-99cf-38207098b444" [weakdeps] +PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [extensions] +PyCallExt = "PyCall" ZygoteExt = "Zygote" [compat] @@ -69,6 +71,7 @@ Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" DelayDiffEq = "bcd4f6db-9728-5f36-b5f7-82caef46ccdb" OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" StochasticDiffEq = "789caeaf-c7a9-5a7d-9973-96adeb23e2a0" @@ -76,4 +79,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [targets] -test = ["Pkg", "SafeTestsets", "Test", "StaticArrays", "StochasticDiffEq", "Aqua", "Zygote"] +test = ["Pkg", "PyCall", "SafeTestsets", "Test", "StaticArrays", "StochasticDiffEq", "Aqua", "Zygote"] diff --git a/ext/PyCallExt.jl b/ext/PyCallExt.jl new file mode 100644 index 000000000..27c638851 --- /dev/null +++ b/ext/PyCallExt.jl @@ -0,0 +1,20 @@ +module PyCallExt + +using PyCall: PyObject, PyAny, pyfunctionret, pyimport, hasproperty +using SciMLBase: solve + +# SciMl uses a function's arity (number of arguments) to determine if it operates in place. +# PyCall does not preserve arity, so we inspect Python functions to find their arity. +function SciMLBase.numargs(f::PyObject) + inspect = pyimport("inspect") + f2 = hasproperty(f, :py_func) ? f.py_func : f + # if `f` is a bound method (i.e., `self.f`), `getfullargspec` includes + # `self` in the `args` list. So, we subtract 1 in that case: + length(first(inspect.getfullargspec(f2))) - inspect.ismethod(f2) +end + +# differential equation solutions can be converted to lists, this tells PyCall not +# to perform that conversion automatically when a solution is returned from `solve` +PyCall.PyObject(::typeof(solve)) = pyfunctionret(solve, Any, Vararg{PyAny}) + +end diff --git a/test/downstream/pycall.jl b/test/downstream/pycall.jl new file mode 100644 index 000000000..ec54f35b3 --- /dev/null +++ b/test/downstream/pycall.jl @@ -0,0 +1,39 @@ +using PyCall, SciMLBase, OrdinaryDiffEq + +@testset "numargs" begin + py""" + def three_arg(a, b, c): + return a + b + c + + def four_arg(a, b, c, d): + return a + b + c + d + + class MyClass: + def three_arg_method(self, a, b, c): + return a + b + c + + def four_arg_method(self, a, b, c, d): + return a + b + c + d + """ + + @test SciMLBase.numargs(py"three_arg") === 3 + @test SciMLBase.numargs(py"four_arg") === 4 + x = py"MyClass()" + @test SciMLBase.numargs(x.three_arg_method) === 3 + @test SciMLBase.numargs(x.four_arg_method) === 4 +end + +@testset "solution handling" begin + py""" + from julia import OrdinaryDiffEq as ode + + def f(u,p,t): + return -u + + u0 = 0.5 + tspan = (0., 1.) + prob = ode.ODEProblem(f, u0, tspan) + sol = ode.solve(prob, ode.Tsit5()) + """ + @test py"sol" isa ODESolution +end diff --git a/test/runtests.jl b/test/runtests.jl index f432e891b..b6f71e38e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -92,5 +92,8 @@ end @time @safetestset "Autodiff Remake" begin include("downstream/remake_autodiff.jl") end + @time @safetestset "PyCall" begin + include("downstream/pycall.jl") + end end end From f6d3f7cce2e4ee2fa7609151d8f9f86168612057 Mon Sep 17 00:00:00 2001 From: Lilith Hafner Date: Sat, 23 Sep 2023 15:52:22 -0500 Subject: [PATCH 02/18] Add more usings --- ext/PyCallExt.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/PyCallExt.jl b/ext/PyCallExt.jl index 27c638851..02d37d632 100644 --- a/ext/PyCallExt.jl +++ b/ext/PyCallExt.jl @@ -1,7 +1,7 @@ module PyCallExt -using PyCall: PyObject, PyAny, pyfunctionret, pyimport, hasproperty -using SciMLBase: solve +using PyCall: PyCall, PyObject, PyAny, pyfunctionret, pyimport, hasproperty +using SciMLBase: SciMLBase, solve # SciMl uses a function's arity (number of arguments) to determine if it operates in place. # PyCall does not preserve arity, so we inspect Python functions to find their arity. From 59909cc96ede3a696e4e96be7ee15ae2cc467db3 Mon Sep 17 00:00:00 2001 From: Lilith Hafner Date: Sat, 23 Sep 2023 15:53:36 -0500 Subject: [PATCH 03/18] Fix capitalization in comment --- ext/PyCallExt.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/PyCallExt.jl b/ext/PyCallExt.jl index 02d37d632..258e2bf20 100644 --- a/ext/PyCallExt.jl +++ b/ext/PyCallExt.jl @@ -3,7 +3,7 @@ module PyCallExt using PyCall: PyCall, PyObject, PyAny, pyfunctionret, pyimport, hasproperty using SciMLBase: SciMLBase, solve -# SciMl uses a function's arity (number of arguments) to determine if it operates in place. +# SciML uses a function's arity (number of arguments) to determine if it operates in place. # PyCall does not preserve arity, so we inspect Python functions to find their arity. function SciMLBase.numargs(f::PyObject) inspect = pyimport("inspect") From f5c538ce8cfa1d4a7369fd9075859b970a32671b Mon Sep 17 00:00:00 2001 From: Christopher Rackauckas Date: Sun, 1 Oct 2023 11:23:12 +0200 Subject: [PATCH 04/18] Update Project.toml --- test/downstream/Project.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/downstream/Project.toml b/test/downstream/Project.toml index db1a40a1c..c5b2c12a9 100644 --- a/test/downstream/Project.toml +++ b/test/downstream/Project.toml @@ -4,6 +4,7 @@ ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78" Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba" OptimizationOptimJL = "36348300-93cb-4f02-beb5-3c3902f8871e" OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" +PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" SciMLSensitivity = "1ed8b502-d754-442c-8d5d-10ac956f44a1" Sundials = "c3572dad-4567-51f8-b174-8c6c989267f4" @@ -13,6 +14,7 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] BoundaryValueDiffEq = "5" ModelingToolkit = "8.37" +PyCall = "1.96" Optimization = "3" OptimizationOptimJL = "0.1" OrdinaryDiffEq = "6.33" From a07934b9a289b9a7a699332f178c46e768709613 Mon Sep 17 00:00:00 2001 From: Christopher Rackauckas Date: Sun, 1 Oct 2023 14:43:34 +0200 Subject: [PATCH 05/18] setup Python downstream testing as a separate test group --- .github/workflows/CI.yml | 1 + test/downstream/Project.toml | 2 -- test/python/Project.toml | 11 +++++++++++ test/{downstream => python}/pycall.jl | 3 +++ test/runtests.jl | 5 ++++- 5 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 test/python/Project.toml rename test/{downstream => python}/pycall.jl (92%) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 55160ee26..35dff9cb4 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -18,6 +18,7 @@ jobs: group: - Core - Downstream + - Python version: - '1' steps: diff --git a/test/downstream/Project.toml b/test/downstream/Project.toml index c5b2c12a9..db1a40a1c 100644 --- a/test/downstream/Project.toml +++ b/test/downstream/Project.toml @@ -4,7 +4,6 @@ ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78" Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba" OptimizationOptimJL = "36348300-93cb-4f02-beb5-3c3902f8871e" OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" -PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" SciMLSensitivity = "1ed8b502-d754-442c-8d5d-10ac956f44a1" Sundials = "c3572dad-4567-51f8-b174-8c6c989267f4" @@ -14,7 +13,6 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] BoundaryValueDiffEq = "5" ModelingToolkit = "8.37" -PyCall = "1.96" Optimization = "3" OptimizationOptimJL = "0.1" OrdinaryDiffEq = "6.33" diff --git a/test/python/Project.toml b/test/python/Project.toml new file mode 100644 index 000000000..a0cee6518 --- /dev/null +++ b/test/python/Project.toml @@ -0,0 +1,11 @@ +[deps] +Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d" +OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" +PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" +SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" + +[compat] +Conda = "1.9" +OrdinaryDiffEq = "6.33" +PyCall = "1.96" +SciMLBase = "2" diff --git a/test/downstream/pycall.jl b/test/python/pycall.jl similarity index 92% rename from test/downstream/pycall.jl rename to test/python/pycall.jl index ec54f35b3..7f9aa9e0e 100644 --- a/test/downstream/pycall.jl +++ b/test/python/pycall.jl @@ -1,4 +1,7 @@ using PyCall, SciMLBase, OrdinaryDiffEq +import Conda +Conda.pip_interop(true) +Conda.pip("install", "julia") @testset "numargs" begin py""" diff --git a/test/runtests.jl b/test/runtests.jl index b6f71e38e..12abb45ee 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -92,8 +92,11 @@ end @time @safetestset "Autodiff Remake" begin include("downstream/remake_autodiff.jl") end + end + + if !is_APPVEYOR && GROUP == "Python" @time @safetestset "PyCall" begin - include("downstream/pycall.jl") + include("python/pycall.jl") end end end From 61a6356e3cd17c3f03a8ee020eb50a74549fcfe2 Mon Sep 17 00:00:00 2001 From: Christopher Rackauckas Date: Sun, 1 Oct 2023 15:54:54 +0200 Subject: [PATCH 06/18] make sure to activate the environment --- test/runtests.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index 12abb45ee..c49d74480 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -15,6 +15,12 @@ function activate_downstream_env() Pkg.instantiate() end +function activate_python_env() + Pkg.activate("python") + Pkg.develop(PackageSpec(path = dirname(@__DIR__))) + Pkg.instantiate() +end + @time begin if GROUP == "Core" || GROUP == "All" @time @safetestset "Aqua" begin @@ -95,6 +101,7 @@ end end if !is_APPVEYOR && GROUP == "Python" + activate_python_env() @time @safetestset "PyCall" begin include("python/pycall.jl") end From d3e2b46359b164f5ef02a8391e4ec07299b20576 Mon Sep 17 00:00:00 2001 From: Lilith Hafner Date: Tue, 3 Oct 2023 14:55:15 -0500 Subject: [PATCH 07/18] Add PyCall to default environment during tests --- test/python/Project.toml | 3 +- test/python/pycall.jl | 84 +++++++++++++++++++++++----------------- 2 files changed, 50 insertions(+), 37 deletions(-) diff --git a/test/python/Project.toml b/test/python/Project.toml index a0cee6518..3d9b5420b 100644 --- a/test/python/Project.toml +++ b/test/python/Project.toml @@ -1,11 +1,10 @@ [deps] -Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d" OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" [compat] -Conda = "1.9" OrdinaryDiffEq = "6.33" PyCall = "1.96" SciMLBase = "2" diff --git a/test/python/pycall.jl b/test/python/pycall.jl index 7f9aa9e0e..08f9bc611 100644 --- a/test/python/pycall.jl +++ b/test/python/pycall.jl @@ -1,42 +1,56 @@ using PyCall, SciMLBase, OrdinaryDiffEq -import Conda -Conda.pip_interop(true) -Conda.pip("install", "julia") -@testset "numargs" begin - py""" - def three_arg(a, b, c): - return a + b + c - - def four_arg(a, b, c, d): - return a + b + c + d +# PyCall only works when PyCall is in the default environment :'( +import Pkg +function with_pycall_in_default_environment(f) + path = Pkg.project().path + Pkg.activate() + install = "PyCall" ∉ keys(Pkg.project().dependencies) + install && Pkg.add("PyCall") + try + f() + finally + install && Pkg.rm("PyCall") + Pkg.activate(path) + end +end - class MyClass: - def three_arg_method(self, a, b, c): +with_pycall_in_default_environment() do + @testset "numargs" begin + py""" + def three_arg(a, b, c): return a + b + c - def four_arg_method(self, a, b, c, d): + def four_arg(a, b, c, d): return a + b + c + d - """ - - @test SciMLBase.numargs(py"three_arg") === 3 - @test SciMLBase.numargs(py"four_arg") === 4 - x = py"MyClass()" - @test SciMLBase.numargs(x.three_arg_method) === 3 - @test SciMLBase.numargs(x.four_arg_method) === 4 -end -@testset "solution handling" begin - py""" - from julia import OrdinaryDiffEq as ode - - def f(u,p,t): - return -u - - u0 = 0.5 - tspan = (0., 1.) - prob = ode.ODEProblem(f, u0, tspan) - sol = ode.solve(prob, ode.Tsit5()) - """ - @test py"sol" isa ODESolution -end + class MyClass: + def three_arg_method(self, a, b, c): + return a + b + c + + def four_arg_method(self, a, b, c, d): + return a + b + c + d + """ + + @test SciMLBase.numargs(py"three_arg") === 3 + @test SciMLBase.numargs(py"four_arg") === 4 + x = py"MyClass()" + @test SciMLBase.numargs(x.three_arg_method) === 3 + @test SciMLBase.numargs(x.four_arg_method) === 4 + end + + @testset "solution handling" begin + py""" + from julia import OrdinaryDiffEq as ode + + def f(u,p,t): + return -u + + u0 = 0.5 + tspan = (0., 1.) + prob = ode.ODEProblem(f, u0, tspan) + sol = ode.solve(prob, ode.Tsit5()) + """ + @test py"sol" isa ODESolution + end +end # with_pycall_in_default_environment From eddd2c884f341062cc75e839f675e16c93b80d1d Mon Sep 17 00:00:00 2001 From: Lilith Hafner Date: Tue, 3 Oct 2023 15:32:18 -0500 Subject: [PATCH 08/18] fixup --- test/python/pycall.jl | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/python/pycall.jl b/test/python/pycall.jl index 08f9bc611..5806f1c72 100644 --- a/test/python/pycall.jl +++ b/test/python/pycall.jl @@ -6,12 +6,18 @@ function with_pycall_in_default_environment(f) path = Pkg.project().path Pkg.activate() install = "PyCall" ∉ keys(Pkg.project().dependencies) - install && Pkg.add("PyCall") + if install + Pkg.add("PyCall") + end + Pkg.activate(path) try f() finally - install && Pkg.rm("PyCall") - Pkg.activate(path) + if install + Pkg.activate() + Pkg.rm("PyCall") + Pkg.activate(path) + end end end From 709006247e1648dbe9430256629985a9d696a33d Mon Sep 17 00:00:00 2001 From: Lilith Hafner Date: Tue, 3 Oct 2023 15:55:31 -0500 Subject: [PATCH 09/18] simplify to try to start biseciont --- test/runtests.jl | 210 ++++++++++++++++++++++++----------------------- 1 file changed, 108 insertions(+), 102 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index c49d74480..4c644cd06 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,109 +1,115 @@ -using Pkg -using SafeTestsets -using Test -using SciMLBase +using PyCall -# https://github.com/JuliaArrays/FillArrays.jl/pull/163 -@test_broken isempty(detect_ambiguities(SciMLBase)) +py""" +import julia +""" -const GROUP = get(ENV, "GROUP", "All") -const is_APPVEYOR = (Sys.iswindows() && haskey(ENV, "APPVEYOR")) +# using Pkg +# using SafeTestsets +# using Test +# using SciMLBase -function activate_downstream_env() - Pkg.activate("downstream") - Pkg.develop(PackageSpec(path = dirname(@__DIR__))) - Pkg.instantiate() -end +# # https://github.com/JuliaArrays/FillArrays.jl/pull/163 +# @test_broken isempty(detect_ambiguities(SciMLBase)) -function activate_python_env() - Pkg.activate("python") - Pkg.develop(PackageSpec(path = dirname(@__DIR__))) - Pkg.instantiate() -end +# const GROUP = get(ENV, "GROUP", "All") +# const is_APPVEYOR = (Sys.iswindows() && haskey(ENV, "APPVEYOR")) -@time begin - if GROUP == "Core" || GROUP == "All" - @time @safetestset "Aqua" begin - include("aqua.jl") - end - @time @safetestset "Display" begin - include("display.jl") - end - @time @safetestset "Existence functions" begin - include("existence_functions.jl") - end - @time @safetestset "Function Building Error Messages" begin - include("function_building_error_messages.jl") - end - @time @safetestset "Solver Missing Error Messages" begin - include("solver_missing_error_messages.jl") - end - @time @safetestset "Integrator interface" begin - include("integrator_tests.jl") - end - @time @safetestset "Table Traits" begin - include("traits.jl") - end - @time @safetestset "Ensemble functionality" begin - include("ensemble_tests.jl") - end - @time @safetestset "DiffEqOperator tests" begin - include("diffeqoperator.jl") - end - @time @safetestset "Solution interface" begin - include("solution_interface.jl") - end - @time @safetestset "DE function conversion" begin - include("convert_tests.jl") - end - @time @safetestset "Performance warnings" begin - include("performance_warnings.jl") - end - @time @safetestset "Problem building tests" begin - include("problem_building_test.jl") - end - end +# function activate_downstream_env() +# Pkg.activate("downstream") +# Pkg.develop(PackageSpec(path = dirname(@__DIR__))) +# Pkg.instantiate() +# end - if !is_APPVEYOR && GROUP == "Downstream" - activate_downstream_env() - @time @safetestset "Ensembles of Zero Length Solutions" begin - include("downstream/ensemble_zero_length.jl") - end - @time @safetestset "Timing first batch when solving Ensembles" begin - include("downstream/ensemble_first_batch.jl") - end - @time @safetestset "solving Ensembles with multiple problems" begin - include("downstream/ensemble_multi_prob.jl") - end - @time @safetestset "Symbol and integer based indexing of interpolated solutions" begin - include("downstream/symbol_indexing.jl") - end - if VERSION >= v"1.8" - @time @safetestset "Symbol and integer based indexing of integrators" begin - include("downstream/integrator_indexing.jl") - end - @time @safetestset "Problem Indexing" begin - include("downstream/problem_interface.jl") - end - @time @safetestset "Solution Indexing" begin - include("downstream/solution_interface.jl") - end - end - @time @safetestset "Unitful interpolations" begin - include("downstream/unitful_interpolations.jl") - end - @time @safetestset "Integer idxs" begin - include("downstream/integer_idxs.jl") - end - @time @safetestset "Autodiff Remake" begin - include("downstream/remake_autodiff.jl") - end - end +# function activate_python_env() +# Pkg.activate("python") +# Pkg.develop(PackageSpec(path = dirname(@__DIR__))) +# Pkg.instantiate() +# end - if !is_APPVEYOR && GROUP == "Python" - activate_python_env() - @time @safetestset "PyCall" begin - include("python/pycall.jl") - end - end -end +# @time begin +# if GROUP == "Core" || GROUP == "All" +# @time @safetestset "Aqua" begin +# include("aqua.jl") +# end +# @time @safetestset "Display" begin +# include("display.jl") +# end +# @time @safetestset "Existence functions" begin +# include("existence_functions.jl") +# end +# @time @safetestset "Function Building Error Messages" begin +# include("function_building_error_messages.jl") +# end +# @time @safetestset "Solver Missing Error Messages" begin +# include("solver_missing_error_messages.jl") +# end +# @time @safetestset "Integrator interface" begin +# include("integrator_tests.jl") +# end +# @time @safetestset "Table Traits" begin +# include("traits.jl") +# end +# @time @safetestset "Ensemble functionality" begin +# include("ensemble_tests.jl") +# end +# @time @safetestset "DiffEqOperator tests" begin +# include("diffeqoperator.jl") +# end +# @time @safetestset "Solution interface" begin +# include("solution_interface.jl") +# end +# @time @safetestset "DE function conversion" begin +# include("convert_tests.jl") +# end +# @time @safetestset "Performance warnings" begin +# include("performance_warnings.jl") +# end +# @time @safetestset "Problem building tests" begin +# include("problem_building_test.jl") +# end +# end + +# if !is_APPVEYOR && GROUP == "Downstream" +# activate_downstream_env() +# @time @safetestset "Ensembles of Zero Length Solutions" begin +# include("downstream/ensemble_zero_length.jl") +# end +# @time @safetestset "Timing first batch when solving Ensembles" begin +# include("downstream/ensemble_first_batch.jl") +# end +# @time @safetestset "solving Ensembles with multiple problems" begin +# include("downstream/ensemble_multi_prob.jl") +# end +# @time @safetestset "Symbol and integer based indexing of interpolated solutions" begin +# include("downstream/symbol_indexing.jl") +# end +# if VERSION >= v"1.8" +# @time @safetestset "Symbol and integer based indexing of integrators" begin +# include("downstream/integrator_indexing.jl") +# end +# @time @safetestset "Problem Indexing" begin +# include("downstream/problem_interface.jl") +# end +# @time @safetestset "Solution Indexing" begin +# include("downstream/solution_interface.jl") +# end +# end +# @time @safetestset "Unitful interpolations" begin +# include("downstream/unitful_interpolations.jl") +# end +# @time @safetestset "Integer idxs" begin +# include("downstream/integer_idxs.jl") +# end +# @time @safetestset "Autodiff Remake" begin +# include("downstream/remake_autodiff.jl") +# end +# end + +# if !is_APPVEYOR && GROUP == "Python" +# activate_python_env() +# @time @safetestset "PyCall" begin +# include("python/pycall.jl") +# end +# end +# end From 4f2ca9b5749dec42e0e14ef5a77a32f6580f9aab Mon Sep 17 00:00:00 2001 From: Lilith Hafner Date: Tue, 3 Oct 2023 16:28:18 -0500 Subject: [PATCH 10/18] more debug --- test/runtests.jl | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 4c644cd06..7f69a9556 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,7 +1,25 @@ using PyCall py""" -import julia +print("AAAAAAAAAA") +import sys +print(sys.executable) +try: + import julia + print("BBBBBBBBBB") +except: + print("CCCCCCCCCC") + +import sys +subprocess.run([sys.executable, '-m', 'pip', 'install', '--user', 'julia']) + +try: + import julia + print("DDDDDDDDDD") +except: + print("EEEEEEEEEE") + +print("FFFFFFFFFF") """ # using Pkg From daaa96f5667de895be19e120f2cb30009ab8851f Mon Sep 17 00:00:00 2001 From: Lilith Hafner Date: Tue, 3 Oct 2023 16:37:36 -0500 Subject: [PATCH 11/18] fixup --- test/runtests.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 7f69a9556..be4bd2ea9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,9 @@ using PyCall +py""" +print("ZZZZZZZZZ") +""" + py""" print("AAAAAAAAAA") import sys @@ -10,7 +14,7 @@ try: except: print("CCCCCCCCCC") -import sys +import subprocess subprocess.run([sys.executable, '-m', 'pip', 'install', '--user', 'julia']) try: From 243a4c07be7b4b208d56743a2b825c2ba4c4a172 Mon Sep 17 00:00:00 2001 From: Lilith Hafner Date: Tue, 3 Oct 2023 17:20:27 -0500 Subject: [PATCH 12/18] more debug foolishness --- test/runtests.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index be4bd2ea9..69fe72a86 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -15,7 +15,7 @@ except: print("CCCCCCCCCC") import subprocess -subprocess.run([sys.executable, '-m', 'pip', 'install', '--user', 'julia']) +subprocess.run([sys.executable, '-m', 'pip', 'install', 'julia']) try: import julia @@ -26,6 +26,10 @@ except: print("FFFFFFFFFF") """ +py""" +import julia +""" + # using Pkg # using SafeTestsets # using Test From 1b9fc827740894d19d6006ef5804bd17eadd3a45 Mon Sep 17 00:00:00 2001 From: Lilith Hafner Date: Thu, 5 Oct 2023 07:00:30 -0500 Subject: [PATCH 13/18] more debug fooling around --- test/python/Project.toml | 3 +- test/python/pycall.jl | 90 ++++++--------- test/runtests.jl | 236 +++++++++++++++++---------------------- 3 files changed, 139 insertions(+), 190 deletions(-) diff --git a/test/python/Project.toml b/test/python/Project.toml index 3d9b5420b..a0cee6518 100644 --- a/test/python/Project.toml +++ b/test/python/Project.toml @@ -1,10 +1,11 @@ [deps] +Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d" OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" -Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" [compat] +Conda = "1.9" OrdinaryDiffEq = "6.33" PyCall = "1.96" SciMLBase = "2" diff --git a/test/python/pycall.jl b/test/python/pycall.jl index 5806f1c72..103db7483 100644 --- a/test/python/pycall.jl +++ b/test/python/pycall.jl @@ -1,62 +1,42 @@ +import Conda +Conda.pip_interop(true) +Conda.pip("install", "julia") using PyCall, SciMLBase, OrdinaryDiffEq -# PyCall only works when PyCall is in the default environment :'( -import Pkg -function with_pycall_in_default_environment(f) - path = Pkg.project().path - Pkg.activate() - install = "PyCall" ∉ keys(Pkg.project().dependencies) - if install - Pkg.add("PyCall") - end - Pkg.activate(path) - try - f() - finally - if install - Pkg.activate() - Pkg.rm("PyCall") - Pkg.activate(path) - end - end -end +@testset "numargs" begin + py""" + def three_arg(a, b, c): + return a + b + c + + def four_arg(a, b, c, d): + return a + b + c + d -with_pycall_in_default_environment() do - @testset "numargs" begin - py""" - def three_arg(a, b, c): + class MyClass: + def three_arg_method(self, a, b, c): return a + b + c - def four_arg(a, b, c, d): + def four_arg_method(self, a, b, c, d): return a + b + c + d + """ + + @test SciMLBase.numargs(py"three_arg") === 3 + @test SciMLBase.numargs(py"four_arg") === 4 + x = py"MyClass()" + @test SciMLBase.numargs(x.three_arg_method) === 3 + @test SciMLBase.numargs(x.four_arg_method) === 4 +end - class MyClass: - def three_arg_method(self, a, b, c): - return a + b + c - - def four_arg_method(self, a, b, c, d): - return a + b + c + d - """ - - @test SciMLBase.numargs(py"three_arg") === 3 - @test SciMLBase.numargs(py"four_arg") === 4 - x = py"MyClass()" - @test SciMLBase.numargs(x.three_arg_method) === 3 - @test SciMLBase.numargs(x.four_arg_method) === 4 - end - - @testset "solution handling" begin - py""" - from julia import OrdinaryDiffEq as ode - - def f(u,p,t): - return -u - - u0 = 0.5 - tspan = (0., 1.) - prob = ode.ODEProblem(f, u0, tspan) - sol = ode.solve(prob, ode.Tsit5()) - """ - @test py"sol" isa ODESolution - end -end # with_pycall_in_default_environment +@testset "solution handling" begin + py""" + from julia import OrdinaryDiffEq as ode + + def f(u,p,t): + return -u + + u0 = 0.5 + tspan = (0., 1.) + prob = ode.ODEProblem(f, u0, tspan) + sol = ode.solve(prob, ode.Tsit5()) + """ + @test py"sol" isa ODESolution +end diff --git a/test/runtests.jl b/test/runtests.jl index 69fe72a86..c49d74480 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,141 +1,109 @@ -using PyCall +using Pkg +using SafeTestsets +using Test +using SciMLBase -py""" -print("ZZZZZZZZZ") -""" +# https://github.com/JuliaArrays/FillArrays.jl/pull/163 +@test_broken isempty(detect_ambiguities(SciMLBase)) -py""" -print("AAAAAAAAAA") -import sys -print(sys.executable) -try: - import julia - print("BBBBBBBBBB") -except: - print("CCCCCCCCCC") +const GROUP = get(ENV, "GROUP", "All") +const is_APPVEYOR = (Sys.iswindows() && haskey(ENV, "APPVEYOR")) -import subprocess -subprocess.run([sys.executable, '-m', 'pip', 'install', 'julia']) +function activate_downstream_env() + Pkg.activate("downstream") + Pkg.develop(PackageSpec(path = dirname(@__DIR__))) + Pkg.instantiate() +end -try: - import julia - print("DDDDDDDDDD") -except: - print("EEEEEEEEEE") +function activate_python_env() + Pkg.activate("python") + Pkg.develop(PackageSpec(path = dirname(@__DIR__))) + Pkg.instantiate() +end -print("FFFFFFFFFF") -""" +@time begin + if GROUP == "Core" || GROUP == "All" + @time @safetestset "Aqua" begin + include("aqua.jl") + end + @time @safetestset "Display" begin + include("display.jl") + end + @time @safetestset "Existence functions" begin + include("existence_functions.jl") + end + @time @safetestset "Function Building Error Messages" begin + include("function_building_error_messages.jl") + end + @time @safetestset "Solver Missing Error Messages" begin + include("solver_missing_error_messages.jl") + end + @time @safetestset "Integrator interface" begin + include("integrator_tests.jl") + end + @time @safetestset "Table Traits" begin + include("traits.jl") + end + @time @safetestset "Ensemble functionality" begin + include("ensemble_tests.jl") + end + @time @safetestset "DiffEqOperator tests" begin + include("diffeqoperator.jl") + end + @time @safetestset "Solution interface" begin + include("solution_interface.jl") + end + @time @safetestset "DE function conversion" begin + include("convert_tests.jl") + end + @time @safetestset "Performance warnings" begin + include("performance_warnings.jl") + end + @time @safetestset "Problem building tests" begin + include("problem_building_test.jl") + end + end -py""" -import julia -""" + if !is_APPVEYOR && GROUP == "Downstream" + activate_downstream_env() + @time @safetestset "Ensembles of Zero Length Solutions" begin + include("downstream/ensemble_zero_length.jl") + end + @time @safetestset "Timing first batch when solving Ensembles" begin + include("downstream/ensemble_first_batch.jl") + end + @time @safetestset "solving Ensembles with multiple problems" begin + include("downstream/ensemble_multi_prob.jl") + end + @time @safetestset "Symbol and integer based indexing of interpolated solutions" begin + include("downstream/symbol_indexing.jl") + end + if VERSION >= v"1.8" + @time @safetestset "Symbol and integer based indexing of integrators" begin + include("downstream/integrator_indexing.jl") + end + @time @safetestset "Problem Indexing" begin + include("downstream/problem_interface.jl") + end + @time @safetestset "Solution Indexing" begin + include("downstream/solution_interface.jl") + end + end + @time @safetestset "Unitful interpolations" begin + include("downstream/unitful_interpolations.jl") + end + @time @safetestset "Integer idxs" begin + include("downstream/integer_idxs.jl") + end + @time @safetestset "Autodiff Remake" begin + include("downstream/remake_autodiff.jl") + end + end -# using Pkg -# using SafeTestsets -# using Test -# using SciMLBase - -# # https://github.com/JuliaArrays/FillArrays.jl/pull/163 -# @test_broken isempty(detect_ambiguities(SciMLBase)) - -# const GROUP = get(ENV, "GROUP", "All") -# const is_APPVEYOR = (Sys.iswindows() && haskey(ENV, "APPVEYOR")) - -# function activate_downstream_env() -# Pkg.activate("downstream") -# Pkg.develop(PackageSpec(path = dirname(@__DIR__))) -# Pkg.instantiate() -# end - -# function activate_python_env() -# Pkg.activate("python") -# Pkg.develop(PackageSpec(path = dirname(@__DIR__))) -# Pkg.instantiate() -# end - -# @time begin -# if GROUP == "Core" || GROUP == "All" -# @time @safetestset "Aqua" begin -# include("aqua.jl") -# end -# @time @safetestset "Display" begin -# include("display.jl") -# end -# @time @safetestset "Existence functions" begin -# include("existence_functions.jl") -# end -# @time @safetestset "Function Building Error Messages" begin -# include("function_building_error_messages.jl") -# end -# @time @safetestset "Solver Missing Error Messages" begin -# include("solver_missing_error_messages.jl") -# end -# @time @safetestset "Integrator interface" begin -# include("integrator_tests.jl") -# end -# @time @safetestset "Table Traits" begin -# include("traits.jl") -# end -# @time @safetestset "Ensemble functionality" begin -# include("ensemble_tests.jl") -# end -# @time @safetestset "DiffEqOperator tests" begin -# include("diffeqoperator.jl") -# end -# @time @safetestset "Solution interface" begin -# include("solution_interface.jl") -# end -# @time @safetestset "DE function conversion" begin -# include("convert_tests.jl") -# end -# @time @safetestset "Performance warnings" begin -# include("performance_warnings.jl") -# end -# @time @safetestset "Problem building tests" begin -# include("problem_building_test.jl") -# end -# end - -# if !is_APPVEYOR && GROUP == "Downstream" -# activate_downstream_env() -# @time @safetestset "Ensembles of Zero Length Solutions" begin -# include("downstream/ensemble_zero_length.jl") -# end -# @time @safetestset "Timing first batch when solving Ensembles" begin -# include("downstream/ensemble_first_batch.jl") -# end -# @time @safetestset "solving Ensembles with multiple problems" begin -# include("downstream/ensemble_multi_prob.jl") -# end -# @time @safetestset "Symbol and integer based indexing of interpolated solutions" begin -# include("downstream/symbol_indexing.jl") -# end -# if VERSION >= v"1.8" -# @time @safetestset "Symbol and integer based indexing of integrators" begin -# include("downstream/integrator_indexing.jl") -# end -# @time @safetestset "Problem Indexing" begin -# include("downstream/problem_interface.jl") -# end -# @time @safetestset "Solution Indexing" begin -# include("downstream/solution_interface.jl") -# end -# end -# @time @safetestset "Unitful interpolations" begin -# include("downstream/unitful_interpolations.jl") -# end -# @time @safetestset "Integer idxs" begin -# include("downstream/integer_idxs.jl") -# end -# @time @safetestset "Autodiff Remake" begin -# include("downstream/remake_autodiff.jl") -# end -# end - -# if !is_APPVEYOR && GROUP == "Python" -# activate_python_env() -# @time @safetestset "PyCall" begin -# include("python/pycall.jl") -# end -# end -# end + if !is_APPVEYOR && GROUP == "Python" + activate_python_env() + @time @safetestset "PyCall" begin + include("python/pycall.jl") + end + end +end From 0542b83aa504c5f8a0b061ea723c008c4ac87884 Mon Sep 17 00:00:00 2001 From: Lilith Hafner Date: Thu, 5 Oct 2023 07:19:58 -0500 Subject: [PATCH 14/18] smarter? debug fooling around --- test/python/pycall.jl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/python/pycall.jl b/test/python/pycall.jl index 103db7483..511869b00 100644 --- a/test/python/pycall.jl +++ b/test/python/pycall.jl @@ -1,8 +1,10 @@ -import Conda -Conda.pip_interop(true) -Conda.pip("install", "julia") using PyCall, SciMLBase, OrdinaryDiffEq +py""" # TODO: upstream this into PyCall +from pip import _internal +_internal.main(['install', 'julia']) +""" + @testset "numargs" begin py""" def three_arg(a, b, c): From 15e115bfd36680b7aad1043bf561bd35391dc3e6 Mon Sep 17 00:00:00 2001 From: Lilith Hafner Date: Thu, 5 Oct 2023 07:53:04 -0500 Subject: [PATCH 15/18] add user packages to sys.path --- test/python/pycall.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/python/pycall.jl b/test/python/pycall.jl index 511869b00..0b03a6231 100644 --- a/test/python/pycall.jl +++ b/test/python/pycall.jl @@ -1,8 +1,9 @@ using PyCall, SciMLBase, OrdinaryDiffEq -py""" # TODO: upstream this into PyCall -from pip import _internal -_internal.main(['install', 'julia']) +py""" # This is a mess because normal site-packages is not writeable in CI +import pip, site, sys +pip._internal.main(['install', '--user', 'julia']) +sys.path.append(site.getusersitepackages()) """ @testset "numargs" begin From 68c1c8c4be52eccecc4412e6b602fa3f5be84bf5 Mon Sep 17 00:00:00 2001 From: Lilith Hafner Date: Thu, 5 Oct 2023 08:01:20 -0500 Subject: [PATCH 16/18] fixup --- test/python/pycall.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/python/pycall.jl b/test/python/pycall.jl index 0b03a6231..feb5a2a16 100644 --- a/test/python/pycall.jl +++ b/test/python/pycall.jl @@ -1,8 +1,9 @@ using PyCall, SciMLBase, OrdinaryDiffEq py""" # This is a mess because normal site-packages is not writeable in CI -import pip, site, sys -pip._internal.main(['install', '--user', 'julia']) +from pip import _internal as _internal +_internal.main(['install', '--user', 'julia']) +import site, sys sys.path.append(site.getusersitepackages()) """ From 5800d62f4d9d45bfe3772f9921f08ce0c11813ab Mon Sep 17 00:00:00 2001 From: Lilith Hafner Date: Thu, 5 Oct 2023 08:34:09 -0500 Subject: [PATCH 17/18] cleanup --- test/python/Project.toml | 2 -- test/python/pycall.jl | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/test/python/Project.toml b/test/python/Project.toml index a0cee6518..8ec377a5f 100644 --- a/test/python/Project.toml +++ b/test/python/Project.toml @@ -1,11 +1,9 @@ [deps] -Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d" OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" [compat] -Conda = "1.9" OrdinaryDiffEq = "6.33" PyCall = "1.96" SciMLBase = "2" diff --git a/test/python/pycall.jl b/test/python/pycall.jl index feb5a2a16..da4f0f51e 100644 --- a/test/python/pycall.jl +++ b/test/python/pycall.jl @@ -1,7 +1,7 @@ using PyCall, SciMLBase, OrdinaryDiffEq py""" # This is a mess because normal site-packages is not writeable in CI -from pip import _internal as _internal +from pip import _internal _internal.main(['install', '--user', 'julia']) import site, sys sys.path.append(site.getusersitepackages()) From 931449c79be71fb4a7afb4d16fec2e70d5cfbc11 Mon Sep 17 00:00:00 2001 From: Lilith Hafner Date: Thu, 5 Oct 2023 08:39:49 -0500 Subject: [PATCH 18/18] switch from pip internals to public API --- test/python/pycall.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/python/pycall.jl b/test/python/pycall.jl index da4f0f51e..26818f98f 100644 --- a/test/python/pycall.jl +++ b/test/python/pycall.jl @@ -1,9 +1,8 @@ using PyCall, SciMLBase, OrdinaryDiffEq py""" # This is a mess because normal site-packages is not writeable in CI -from pip import _internal -_internal.main(['install', '--user', 'julia']) -import site, sys +import subprocess, sys, site +subprocess.run([sys.executable, '-m', 'pip', 'install', '--user', 'julia']) sys.path.append(site.getusersitepackages()) """