diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml new file mode 100644 index 000000000..b7053c0f4 --- /dev/null +++ b/.github/workflows/draft-pdf.yml @@ -0,0 +1,24 @@ +name: Draft PDF +on: [push] + +jobs: + paper: + runs-on: ubuntu-latest + name: Paper Draft + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build draft PDF + uses: openjournals/openjournals-draft-action@master + with: + journal: joss + # This should be the path to the paper within your repo. + paper-path: paper/paper.md + - name: Upload + uses: actions/upload-artifact@v4 + with: + name: paper + # This is the output path where Pandoc will write the compiled + # PDF. Note, this should be the same directory as the input + # paper.md + path: paper/paper.pdf diff --git a/paper/paper.bib b/paper/paper.bib new file mode 100644 index 000000000..b7f7ff5b8 --- /dev/null +++ b/paper/paper.bib @@ -0,0 +1,370 @@ +@InProceedings{krämer2021highdim, + title = {Probabilistic {ODE} Solutions in Millions of Dimensions}, + author = {Kr{\"a}mer, Nicholas and Bosch, Nathanael and Schmidt, + Jonathan and Hennig, Philipp}, + booktitle = {Proceedings of the 39th International Conference on Machine + Learning}, + pages = {11634--11649}, + year = 2022, + editor = {Chaudhuri, Kamalika and Jegelka, Stefanie and Song, Le and + Szepesvari, Csaba and Niu, Gang and Sabato, Sivan}, + volume = 162, + series = {Proceedings of Machine Learning Research}, + month = {17--23 Jul}, + publisher = {PMLR}, + pdf = {https://proceedings.mlr.press/v162/kramer22b/kramer22b.pdf}, + url = {https://proceedings.mlr.press/v162/kramer22b.html}, +} + +@InProceedings{tronarp2022fenrir, + title = {Fenrir: Physics-Enhanced Regression for Initial Value + Problems}, + author = {Tronarp, Filip and Bosch, Nathanael and Hennig, Philipp}, + AUTHOR+an = {2=highlight}, + booktitle = {Proceedings of the 39th International Conference on Machine + Learning}, + pages = {21776--21794}, + year = 2022, + editor = {Chaudhuri, Kamalika and Jegelka, Stefanie and Song, Le and + Szepesvari, Csaba and Niu, Gang and Sabato, Sivan}, + volume = 162, + series = {Proceedings of Machine Learning Research}, + month = {17--23 Jul}, + publisher = {PMLR}, + pdf = {https://proceedings.mlr.press/v162/tronarp22a/tronarp22a.pdf}, + url = {https://proceedings.mlr.press/v162/tronarp22a.html}, +} + +@inproceedings{bosch2023probabilistic, + title = {Probabilistic Exponential Integrators}, + author = {Nathanael Bosch and Philipp Hennig and Filip Tronarp}, + booktitle = {Thirty-seventh Conference on Neural Information Processing + Systems}, + year = 2023, + url = {https://openreview.net/forum?id=2dx5MNs2Ip}, +} + +@inproceedings{beck2024diffusion, + title = {Diffusion Tempering Improves Parameter Estimation with + Probabilistic Integrators for Ordinary Differential Equations}, + author = {Jonas Beck and Nathanael Bosch and Michael Deistler and Kyra + L. Kadhim and Jakob H. Macke and Philipp Hennig and Philipp + Berens}, + booktitle = {Forty-first International Conference on Machine Learning}, + year = 2024, + url = {https://openreview.net/forum?id=43HZG9zwaj} +} + +@misc{wenger2021probnum, + title = {ProbNum: Probabilistic Numerics in {P}ython}, + author = {Jonathan Wenger and Nicholas Krämer and Marvin Pförtner and + Jonathan Schmidt and Nathanael Bosch and Nina Effenberger and + Johannes Zenn and Alexandra Gessner and Toni Karvonen and + François-Xavier Briol and Maren Mahsereci and Philipp Hennig}, + year = 2021, + eprint = {2112.02100}, + archivePrefix ={arXiv}, + primaryClass = {cs.MS}, + doi = "10.48550/arXiv.2112.02100", + url = "https://doi.org/10.48550/arXiv.2112.02100", +} + +@InProceedings{dalton2024, + title = {Data-Adaptive Probabilistic Likelihood Approximation for + Ordinary Differential Equations}, + author = {Wu, Mohan and Lysy, Martin}, + booktitle = {Proceedings of The 27th International Conference on Artificial + Intelligence and Statistics}, + pages = {1018--1026}, + year = 2024, + editor = {Dasgupta, Sanjoy and Mandt, Stephan and Li, Yingzhen}, + volume = 238, + series = {Proceedings of Machine Learning Research}, + month = {02--04 May}, + publisher = {PMLR}, + pdf = {https://proceedings.mlr.press/v238/wu24b/wu24b.pdf}, + url = {https://proceedings.mlr.press/v238/wu24b.html}, +} + +@article{rackauckas2017differentialequations, + title = {{DifferentialEquations.jl} – {A} Performant and Feature-Rich + Ecosystem for Solving Differential Equations in {J}ulia}, + author = {Rackauckas, Christopher and Nie, Qing}, + journal = {Journal of Open Research Software}, + volume = 5, + number = 1, + year = 2017, + publisher = {Ubiquity Press}, + doi = "10.5334/jors.151", + url = "https://doi.org/10.5334/jors.151", +} + +@misc{probdiffeq, + author = {Kr{\"a}mer, Nicholas}, + title = {probdiffeq: Probabilistic solvers for differential equations + in JAX}, + year = 2023, + publisher = {GitHub}, + journal = {GitHub repository}, + url = {https://github.com/pnkraemer/probdiffeq} +} + +@misc{jax2018github, + author = {James Bradbury and Roy Frostig and Peter Hawkins and Matthew + James Johnson and Chris Leary and Dougal Maclaurin and George + Necula and Adam Paszke and Jake Vander{P}las and Skye + Wanderman-{M}ilne and Qiao Zhang}, + title = {{JAX}: composable transformations of {P}ython+{N}um{P}y + programs}, + version = {0.2.5}, + year = 2018, +} + +@article{schober19, + author = "Schober, Michael and S{\"a}rkk{\"a}, Simo and Hennig, Philipp", + title = "A probabilistic model for the numerical solution of initial + value problems", + journal = "Statistics and Computing", + year = 2019, + month = "Jan", + day = 01, + volume = 29, + number = 1, + pages = "99--122", + issn = "1573-1375", + doi = "10.1007/s11222-017-9798-7", + url = "https://doi.org/10.1007/s11222-017-9798-7" +} + +@article{tronarp19, + author = {Filip Tronarp and Hans Kersting and Simo S{\"{a}}rkk{\"{a}} + and Philipp Hennig}, + title = {Probabilistic solutions to ordinary differential equations as + nonlinear {B}ayesian filtering: a new perspective}, + year = 2019, + volume = 29, + number = 6, + pages = {1297-1315}, + doi = {10.1007/s11222-019-09900-1}, + url = {https://doi.org/10.1007/s11222-019-09900-1}, + journal = {Statistics and Computing}, + timestamp = {Wed, 25 Mar 2020 09:31:58 +0100}, + biburl = {https://dblp.org/rec/journals/sac/TronarpKSH19.bib}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} + +@article{kersting20, + author = {Kersting, Hans and Sullivan, T. J. and Hennig, Philipp}, + title = {Convergence rates of {G}aussian ODE filters}, + journal = {Statistics and Computing}, + year = 2020, + month = {Nov}, + day = 01, + volume = 30, + number = 6, + pages = {1791-1816}, + issn = {1573-1375}, + doi = {10.1007/s11222-020-09972-4}, + url = {https://doi.org/10.1007/s11222-020-09972-4} +} + +@article{tronarp21, + author = {Tronarp, Filip and S{\"a}rkk{\"a}, Simo and Hennig, Philipp}, + title = {Bayesian ODE solvers: the maximum a posteriori estimate}, + journal = {Statistics and Computing}, + year = 2021, + month = {Mar}, + day = 03, + volume = 31, + number = 3, + pages = 23, + issn = {1573-1375}, + doi = {10.1007/s11222-021-09993-7}, + url = {https://doi.org/10.1007/s11222-021-09993-7} +} + +@misc{bosch2023parallelintime, + title = {Parallel-in-Time Probabilistic Numerical {ODE} Solvers}, + author = {Nathanael Bosch and Adrien Corenflos and Fatemeh Yaghoobi and + Filip Tronarp and Philipp Hennig and Simo Särkkä}, + year = 2023, + eprint = {2310.01145}, + archivePrefix ={arXiv}, + primaryClass = {math.NA}, + doi = {10.48550/arXiv.2310.01145}, + url = {https://doi.org/10.48550/arXiv.2310.01145} +} + +@InProceedings{kersting20invprob, + title = {Differentiable Likelihoods for Fast Inversion of + ’{L}ikelihood-Free’ Dynamical Systems}, + author = {Kersting, Hans and Kr{\"a}mer, Nicholas and Schiegg, Martin + and Daniel, Christian and Tiemann, Michael and Hennig, + Philipp}, + booktitle = {Proceedings of the 37th International Conference on Machine + Learning}, + pages = {5198--5208}, + year = 2020, + editor = {Hal Daumé III and Aarti Singh}, + volume = 119, + series = {Proceedings of Machine Learning Research}, + month = {13--18 Jul}, + publisher = {PMLR}, + pdf = {http://proceedings.mlr.press/v119/kersting20a/kersting20a.pdf}, + url = {http://proceedings.mlr.press/v119/kersting20a.html}, +} + +@inproceedings{schmidt21, + author = {Schmidt, Jonathan and Kr\"{a}mer, Nicholas and Hennig, + Philipp}, + booktitle = {Advances in Neural Information Processing Systems}, + editor = {M. Ranzato and A. Beygelzimer and Y. Dauphin and P.S. Liang + and J. Wortman Vaughan}, + pages = {12374--12385}, + publisher = {Curran Associates, Inc.}, + title = {A Probabilistic State Space Model for Joint Inference from + Differential Equations and Data}, + url = + {https://proceedings.neurips.cc/paper/2021/file/6734fa703f6633ab896eecbdfad8953a-Paper.pdf}, + volume = 34, + year = 2021, +} + +@inproceedings{kraemer202bvp, + author = {Kr\"{a}mer, Nicholas and Hennig, Philipp}, + booktitle = {Advances in Neural Information Processing Systems}, + editor = {M. Ranzato and A. Beygelzimer and Y. Dauphin and P.S. Liang + and J. Wortman Vaughan}, + pages = {11160--11171}, + publisher = {Curran Associates, Inc.}, + title = {Linear-Time Probabilistic Solution of Boundary Value Problems}, + url = + {https://papers.nips.cc/paper/2021/hash/5ca3e9b122f61f8f06494c97b1afccf3-Abstract.html}, + volume = 34, + year = 2021, +} + +@InProceedings{kraemer22mol, + title = { Probabilistic Numerical Method of Lines for Time-Dependent + Partial Differential Equations }, + author = {Kr\"amer, Nicholas and Schmidt, Jonathan and Hennig, Philipp}, + booktitle = {Proceedings of The 25th International Conference on Artificial + Intelligence and Statistics}, + pages = {625--639}, + year = 2022, + editor = {Camps-Valls, Gustau and Ruiz, Francisco J. R. and Valera, + Isabel}, + volume = 151, + series = {Proceedings of Machine Learning Research}, + month = {28--30 Mar}, + publisher = {PMLR}, + pdf = {https://proceedings.mlr.press/v151/kramer22a/kramer22a.pdf}, + url = {https://proceedings.mlr.press/v151/kramer22a.html}, +} + +@InProceedings{bosch22pick, + title = { Pick-and-Mix Information Operators for Probabilistic {ODE} + Solvers }, + author = {Bosch, Nathanael and Tronarp, Filip and Hennig, Philipp}, + booktitle = {Proceedings of The 25th International Conference on Artificial + Intelligence and Statistics}, + pages = {10015--10027}, + year = 2022, + editor = {Camps-Valls, Gustau and Ruiz, Francisco J. R. and Valera, + Isabel}, + volume = 151, + series = {Proceedings of Machine Learning Research}, + month = {28--30 Mar}, + publisher = {PMLR}, + pdf = {https://proceedings.mlr.press/v151/bosch22a/bosch22a.pdf}, + url = {https://proceedings.mlr.press/v151/bosch22a.html}, +} + +@InProceedings{bosch21capos, + title = { Calibrated Adaptive Probabilistic {ODE} Solvers }, + author = {Bosch, Nathanael and Hennig, Philipp and Tronarp, Filip}, + booktitle = {Proceedings of The 24th International Conference on Artificial + Intelligence and Statistics}, + pages = {3466--3474}, + year = 2021, + editor = {Banerjee, Arindam and Fukumizu, Kenji}, + volume = 130, + series = {Proceedings of Machine Learning Research}, + month = {13--15 Apr}, + publisher = {PMLR}, + pdf = {http://proceedings.mlr.press/v130/bosch21a/bosch21a.pdf}, + url = {http://proceedings.mlr.press/v130/bosch21a.html}, +} + +@article{kraemer24stableimp, + author = {Nicholas Kr{{\"a}}mer and Philipp Hennig}, + title = {Stable Implementation of Probabilistic ODE Solvers}, + journal = {Journal of Machine Learning Research}, + year = 2024, + volume = 25, + number = 111, + pages = {1--29}, + url = {http://jmlr.org/papers/v25/20-1423.html} +} + +@article{julia, + doi = {10.1137/141000671}, + url = {https://doi.org/10.1137%2F141000671}, + year = 2017, + month = {jan}, + publisher = {Society for Industrial {\&} Applied Mathematics ({SIAM})}, + volume = 59, + number = 1, + pages = {65--98}, + author = {Jeff Bezanson and Alan Edelman and Stefan Karpinski and Viral + B. Shah}, + title = {Julia: A Fresh Approach to Numerical Computing}, + journal = {{SIAM} Review} +} + +@Article{numpy, + title = {Array programming with {NumPy}}, + author = {Charles R. Harris and K. Jarrod Millman and St{\'{e}}fan J. + van der Walt and Ralf Gommers and Pauli Virtanen and David + Cournapeau and Eric Wieser and Julian Taylor and Sebastian + Berg and Nathaniel J. Smith and Robert Kern and Matti Picus + and Stephan Hoyer and Marten H. van Kerkwijk and Matthew + Brett and Allan Haldane and Jaime Fern{\'{a}}ndez del + R{\'{i}}o and Mark Wiebe and Pearu Peterson and Pierre + G{\'{e}}rard-Marchant and Kevin Sheppard and Tyler Reddy and + Warren Weckesser and Hameer Abbasi and Christoph Gohlke and + Travis E. Oliphant}, + year = {2020}, + month = sep, + journal = {Nature}, + volume = {585}, + number = {7825}, + pages = {357--362}, + doi = {10.1038/s41586-020-2649-2}, + publisher = {Springer Science and Business Media {LLC}}, + url = {https://doi.org/10.1038/s41586-020-2649-2} +} + +@ARTICLE{scipy, + author = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and + Haberland, Matt and Reddy, Tyler and Cournapeau, David and + Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and + Bright, Jonathan and {van der Walt}, St{\'e}fan J. and + Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and + Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and + Kern, Robert and Larson, Eric and Carey, C J and + Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and + {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and + Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and + Harris, Charles R. and Archibald, Anne M. and + Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and + {van Mulbregt}, Paul and {SciPy 1.0 Contributors}}, + title = {{{SciPy} 1.0: Fundamental Algorithms for Scientific + Computing in Python}}, + journal = {Nature Methods}, + year = {2020}, + volume = {17}, + pages = {261--272}, + adsurl = {https://rdcu.be/b08Wh}, + doi = {10.1038/s41592-019-0686-2}, +} \ No newline at end of file diff --git a/paper/paper.md b/paper/paper.md new file mode 100644 index 000000000..c8b801a7c --- /dev/null +++ b/paper/paper.md @@ -0,0 +1,94 @@ +--- +title: 'ProbNumDiffEq.jl: Probabilistic Numerical Solvers for Ordinary Differential Equations in Julia' +tags: + - Julia + - probabilistic numerics + - differential equations + - Bayesian filtering and smoothing + - simulation +authors: + - name: Nathanael Bosch + orcid: 0000-0003-0139-4622 + affiliation: 1 +affiliations: + - name: Tübingen AI Center, University of Tübingen, Germany + index: 1 +date: 17 July 2024 +bibliography: paper.bib +--- + +# Summary + +Probabilistic numerical solvers have emerged as an efficient framework for simulation, uncertainty quantification, and inference in dynamical systems. +In comparison to traditional numerical methods, which approximate the true trajectory of a system only by a single point estimate, probabilistic numerical solvers compute a _distribution_ over the true unknown solution of the given differential equation +and thereby provide information about the numerical error incurred during the computation. +ProbNumDiffEq.jl implements such probabilistic numerical solvers for ordinary differential equations (ODEs) and differential-algebraic equations (DAEs) in the Julia programming language [@julia] within the DifferentialEquations.jl ecosystem [@rackauckas2017differentialequations]. + +More concretely, ProbNumDiffEq.jl provides a range of probabilistic numerical solvers for ordinary differential equations based on Bayesian filtering and smoothing, +which have emerged as a particularly efficient and flexible class of methods for solving ODEs [@schober19; @kersting20; @tronarp19]. +These so-called "ODE filters" have known polynomial convergence rates +[@kersting20; @tronarp21] +and numerical stability properties (such as A-stability or L-stability) +[@tronarp19; @bosch2023probabilistic], +their computational complexity is comparable to traditional numerical methods +[@krämer2021highdim], +they are applicable to a range of numerical differential equation problems +[@kraemer202bvp; @kraemer22mol; @bosch22pick], +and they can be formulated parallel-in-time +[@bosch2023parallelintime]. +ODE filters also provide a natural framework for ODE parameter inference +[@kersting20invprob; @tronarp2022fenrir; @schmidt21; @beck2024diffusion]. +ProbNumDiffEq.jl implements many of the methods referenced above and packages them in a software library with the aim to be easy-to-use, feature-rich, well-documented, and efficiently implemented. + +# Statement of need + +Filtering-based probabilistic numerical ODE solvers have been an active field of research for the past decade, but their application in practical simulation and inference problems has been limited. +ProbNumDiffEq.jl aims to bridge this gap. +ProbNumDiffEq.jl implements probabilistic numerical methods as performant, documented, and easy-to-use ODE solvers inside the well-established DifferentialEquations.jl ecosystem [@rackauckas2017differentialequations]. +Thereby, the package benefits from the extensive testing, documentation, performance optimization, and functionality that DifferentialEquations.jl provides. +Users can easily find help and examples regarding many features that are not particular to ProbNumDiffEq.jl in the DifferentialEquations.jl documentation, +and we provide additional examples and tutorials specific to the probabilistic solvers in the ProbNumDiffEq.jl documentation. +We believe that this deep integration within DifferentialEquations.jl is a key feature to attract users to probabilistic numerics by enabling the use of probabilistic ODE solvers as drop-in replacements for traditional ODE solvers. + +On the other hand, ProbNumDiffEq.jl also aims to accelerate the development of new probabilistic numerical ODE solvers by providing a solid foundation to both build on and compare against. +Several publications have been developed with ProbNumDiffEq.jl, including contributions on +step-size adaptation and calibration of these solvers [@bosch21capos], +energy-preserving solvers and DAE solvers [@bosch22pick], +probabilistic exponential integrators [@bosch2023probabilistic], +and novel parameter inference algorithms [@tronarp2022fenrir; @beck2024diffusion]. +We also hope that by providing documented and performant implementations of published algorithms, we facilitate researchers to use these methods as baselines when developing new numerical solvers. + +ProbNumDiffEq.jl is also the only software package in Julia, at the time of writing, that provides a comprehensive set of probabilistic numerical ODE solvers. +Outside of Julia, two other software packages provide a similar functionality. +ProbNum [@wenger2021probnum] +is a Python package that implements probabilistic numerical methods for various numerical problems, including linear systems, quadrature, and ODEs. +ProbNum particularly aims to facilitate rapid experimentation and accelerate the development of new methods [@wenger2021probnum]. +It is therefore broader in scope and provides functionality not covered by ProbNumDiffEq.jl, but it also lacks some of the specialized ODE solvers available in ProbNumDiffEq.jl. +In addition, with its reliance on Python and NumPy [@numpy] and the lack of just-in-time compilation, it is also generally less performant. +ProbDiffEq [@probdiffeq] +is a probabilistic numerical ODE solver package built on JAX. +At the time of writing, it provides a very similar set of ODE solvers as ProbNumDiffEq.jl with the addition of certain filtering and smoothing methods and the lack of certain specialized ODE solvers---but as both ProbDiffEq and ProbNumDiffEq.jl are under active development, this might change in the future. +By building on JAX and leveraging its just-in-time compilation capabilities, ProbDiffEq provides ODE solvers with similar performance as those implemented in ProbNumDiffEq.jl (shown through benchmarks in both packages comparing to SciPy [@scipy]). +In summary, ProbNumDiffEq.jl provides one of the most feature-rich and performant probabilistic numerical ODE solver packages currently available and is the only one built on the Julia programming language. + +# Acknowledgements + +The author gratefully acknowledges co-funding by the European Union (ERC, ANUBIS, 101123955. Views and opinions expressed are however those of the author(s) only and do not necessarily reflect those of the European Union or the European Research Council. Neither the European Union nor the granting authority can be held responsible for them). +The author thanks the International Max Planck Research School for Intelligent Systems (IMPRS-IS) for their support. + +I am grateful to Philipp Hennig for his support throughout the development of this package. +I thank Nicholas Krämer, Filip Tronarp, and Jonathan Schmidt for many valuable discussions about probabilistic numerical ODE solvers and their correct, efficient, and elegant implementation. +I also thank Christopher Rackauckas for support and feedback on how to integrate ProbNumDiffEq.jl with the DifferentialEquations.jl ecosystem and for including ProbNumDiffEq.jl into the testing pipeline of OrdinaryDiffEq.jl. +I acknowledge contributions from +Pietro Monticone (\@pitmonticone), +Vedant Puri (\@vpuri3), +Tim Holy (\@timholy), +Daniel González Arribas (\@DaniGlez), +David Widmann (\@devmotion), +Christopher Rackauckas (\@ChrisRackauckas), +Qingyu Qu (\@ErikQQY), +Cornelius Roemer (\@corneliusroemer), +and Jose Storopoli (\@storopoli). + + +# References