diff --git a/Project.toml b/Project.toml index d37c0548a6a..06fd29ba590 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Trixi" uuid = "a7f1ee26-1774-49b1-8366-f1abc58fbfcb" authors = ["Michael Schlottke-Lakemper ", "Gregor Gassner ", "Hendrik Ranocha ", "Andrew R. Winters ", "Jesse Chan "] -version = "0.5.42-pre" +version = "0.5.43-pre" [deps] CodeTracking = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" diff --git a/README.md b/README.md index 63540b1f640..c177ad2347f 100644 --- a/README.md +++ b/README.md @@ -17,16 +17,6 @@

-*** -**Trixi.jl at JuliaCon 2023**
-At this year's JuliaCon, we will be present with an online contribution that involves Trixi.jl: - -* [Scaling Trixi.jl to more than 10,000 cores using MPI](https://pretalx.com/juliacon2023/talk/PC8PZ8/), - 27th July 2023, 10:30–11:30 (US/Eastern), 32-G449 (Kiva) - -We are looking forward to seeing you there ♥️ -*** - **Trixi.jl** is a numerical simulation framework for hyperbolic conservation laws written in [Julia](https://julialang.org). A key objective for the framework is to be useful to both scientists and students. Therefore, next to diff --git a/docs/literate/make.jl b/docs/literate/make.jl index b620f85c975..a04d8a0b333 100644 --- a/docs/literate/make.jl +++ b/docs/literate/make.jl @@ -51,12 +51,25 @@ function create_tutorials(files) # Run tests on all tutorial files @testset "TrixiTutorials" begin for (i, (title, filename)) in enumerate(files) + # Evaluate each tutorial in its own module to avoid leaking of + # function/variable names, polluting the namespace of later tutorials + # by stuff defined in earlier tutorials. if filename isa Vector # Several files of one topic for j in eachindex(filename) - @testset "$(filename[j][2][2])" begin include(joinpath(repo_src, filename[j][2][1], filename[j][2][2])) end + mod = gensym(filename[j][2][2]) + @testset "$(filename[j][2][2])" begin + @eval module $mod + include(joinpath($repo_src, $(filename[j][2][1]), $(filename[j][2][2]))) + end + end end else # Single files - @testset "$title" begin include(joinpath(repo_src, filename)) end + mod = gensym(title) + @testset "$title" begin + @eval module $mod + include(joinpath($repo_src, $filename)) + end + end end end end diff --git a/docs/literate/src/files/custom_semidiscretization.jl b/docs/literate/src/files/custom_semidiscretization.jl new file mode 100644 index 00000000000..fd432fb0826 --- /dev/null +++ b/docs/literate/src/files/custom_semidiscretization.jl @@ -0,0 +1,324 @@ +#src # Custom semidiscretizations + +# As described in the [overview section](@ref overview-semidiscretizations), +# semidiscretizations are high-level descriptions of spatial discretizations +# in Trixi.jl. Trixi.jl's main focus is on hyperbolic conservation +# laws represented in a [`SemidiscretizationHyperbolic`](@ref). +# Hyperbolic-parabolic problems based on the advection-diffusion equation or +# the compressible Navier-Stokes equations can be represented in a +# [`SemidiscretizationHyperbolicParabolic`](@ref). This is described in the +# [basic tutorial on parabolic terms](@ref parabolic_terms) and its extension to +# [custom parabolic terms](@ref adding_new_parabolic_terms). +# In this tutorial, we will describe how these semidiscretizations work and how +# they can be used to create custom semidiscretizations involving also other tasks. + + +# ## Overview of the right-hand side evaluation + +# The semidiscretizations provided by Trixi.jl are set up to create `ODEProblem`s from the +# [SciML ecosystem for ordinary differential equations](https://diffeq.sciml.ai/latest/). +# In particular, a spatial semidiscretization can be wrapped in an ODE problem +# using [`semidiscretize`](@ref), which returns an `ODEProblem`. This `ODEProblem` +# bundles an initial condition, a right-hand side (RHS) function, the time span, +# and possible parameters. The `ODEProblem`s created by Trixi.jl use the semidiscretization +# passed to [`semidiscretize`](@ref) as a parameter. +# For a [`SemidiscretizationHyperbolic`](@ref), the `ODEProblem` wraps +# `Trixi.rhs!` as ODE RHS. +# For a [`SemidiscretizationHyperbolicParabolic`](@ref), Trixi.jl +# uses a `SplitODEProblem` combining `Trixi.rhs_parabolic!` for the +# (potentially) stiff part and `Trixi.rhs!` for the other part. + + +# ## Standard Trixi.jl setup + +# In this tutorial, we will consider the linear advection equation +# with source term +# ```math +# \partial_t u(t,x) + \partial_x u(t,x) = -\exp(-t) \sin\bigl(\pi (x - t) \bigr) +# ``` +# with periodic boundary conditions in the domain `[-1, 1]` as a +# model problem. +# The initial condition is +# ```math +# u(0,x) = \sin(\pi x). +# ``` +# The source term results in some damping and the analytical solution +# ```math +# u(t,x) = \exp(-t) \sin\bigl(\pi (x - t) \bigr). +# ``` +# First, we discretize this equation using the standard functionality +# of Trixi.jl. + +using Trixi, OrdinaryDiffEq, Plots + +# The linear scalar advection equation is already implemented in +# Trixi.jl as [`LinearScalarAdvectionEquation1D`](@ref). We construct +# it with an advection velocity `1.0`. + +equations = LinearScalarAdvectionEquation1D(1.0) + +# Next, we use a standard [`DGSEM`](@ref) solver. + +solver = DGSEM(polydeg = 3) + +# We create a simple [`TreeMesh`](@ref) in 1D. + +coordinates_min = (-1.0,) +coordinates_max = (+1.0,) +mesh = TreeMesh(coordinates_min, coordinates_max; + initial_refinement_level = 4, + n_cells_max = 10^4, + periodicity = true) + +# We wrap everything in in a semidiscretization and pass the source +# terms as a standard Julia function. Please note that Trixi.jl uses +# `SVector`s from +# [StaticArrays.jl](https://github.com/JuliaArrays/StaticArrays.jl) +# to store the conserved variables `u`. Thus, the return value of the +# source terms must be wrapped in an `SVector` - even if we consider +# just a scalar problem. + +function initial_condition(x, t, equations) + return SVector(exp(-t) * sinpi(x[1] - t)) +end + +function source_terms_standard(u, x, t, equations) + return -initial_condition(x, t, equations) +end + +semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition, + solver; + source_terms = source_terms_standard) + +# Now, we can create the `ODEProblem`, solve the resulting ODE +# using a time integration method from +# [OrdinaryDiffEq.jl](https://github.com/SciML/OrdinaryDiffEq.jl), +# and visualize the numerical solution at the final time using +# [Plots.jl](https://github.com/JuliaPlots/Plots.jl). + +tspan = (0.0, 3.0) +ode = semidiscretize(semi, tspan) + +sol = solve(ode, RDPK3SpFSAL49(); ode_default_options()...) + +plot(sol; label = "numerical sol.", legend = :topright) + +# We can also plot the analytical solution for comparison. +# Since Trixi.jl uses `SVector`s for the variables, we take their `first` +# (and only) component to get the scalar value for manual plotting. + +let + x = range(-1.0, 1.0; length = 200) + plot!(x, first.(initial_condition.(x, sol.t[end], equations)), + label = "analytical sol.", linestyle = :dash, legend = :topright) +end + +# We can also add the initial condition to the plot. + +plot!(sol.u[1], semi, label = "u0", linestyle = :dot, legend = :topleft) + +# You can of course also use some +# [callbacks](https://trixi-framework.github.io/Trixi.jl/stable/callbacks/) +# provided by Trixi.jl as usual. + +summary_callback = SummaryCallback() +analysis_interval = 100 +analysis_callback = AnalysisCallback(semi; interval = analysis_interval) +alive_callback = AliveCallback(; analysis_interval) +callbacks = CallbackSet(summary_callback, + analysis_callback, + alive_callback) + +sol = solve(ode, RDPK3SpFSAL49(); + ode_default_options()..., callback = callbacks) +summary_callback() + + +# ## Using a custom ODE right-hand side function + +# Next, we will solve the same problem but use our own ODE RHS function. +# To demonstrate this, we will artificially create a global variable +# containing the current time of the simulation. + +const GLOBAL_TIME = Ref(0.0) + +function source_terms_custom(u, x, t, equations) + t = GLOBAL_TIME[] + return -initial_condition(x, t, equations) +end + +# Next, we create our own RHS function to update the global time of +# the simulation before calling the RHS function from Trixi.jl. + +function rhs_source_custom!(du_ode, u_ode, semi, t) + GLOBAL_TIME[] = t + Trixi.rhs!(du_ode, u_ode, semi, t) +end + +# Next, we create an `ODEProblem` manually copying over the data from +# the one we got from [`semidiscretize`](@ref) earlier. + +ode_source_custom = ODEProblem(rhs_source_custom!, + ode.u0, + ode.tspan, + ode.p #= semi =#) +sol_source_custom = solve(ode_source_custom, RDPK3SpFSAL49(); + ode_default_options()...) + +plot(sol_source_custom; label = "numerical sol.") +let + x = range(-1.0, 1.0; length = 200) + plot!(x, first.(initial_condition.(x, sol_source_custom.t[end], equations)), + label = "analytical sol.", linestyle = :dash, legend = :topleft) +end +plot!(sol_source_custom.u[1], semi, label = "u0", linestyle = :dot, legend = :topleft) + +# This also works with callbacks as usual. + +summary_callback = SummaryCallback() +analysis_interval = 100 +analysis_callback = AnalysisCallback(semi; interval = analysis_interval) +alive_callback = AliveCallback(; analysis_interval) +callbacks = CallbackSet(summary_callback, + analysis_callback, + alive_callback) + +sol = solve(ode_source_custom, RDPK3SpFSAL49(); + ode_default_options()..., callback = callbacks) +summary_callback() + + +# ## Setting up a custom semidiscretization + +# Using a global constant is of course not really nice from a software +# engineering point of view. Thus, it can often be useful to collect +# additional data in the parameters of the `ODEProblem`. Thus, it is +# time to create our own semidiscretization. Here, we create a small +# wrapper of a standard semidiscretization of Trixi.jl and the current +# global time of the simulation. + +struct CustomSemidiscretization{Semi, T} <: Trixi.AbstractSemidiscretization + semi::Semi + t::T +end + +semi_custom = CustomSemidiscretization(semi, Ref(0.0)) + +# To get pretty printing in the REPL, you can consider specializing +# +# - `Base.show(io::IO, parameters::CustomSemidiscretization)` +# - `Base.show(io::IO, ::MIME"text/plain", parameters::CustomSemidiscretization)` +# +# for your custom semidiscretiation. + +# Next, we create our own source terms that use the global time stored +# in the custom semidiscretiation. + +source_terms_custom_semi = let semi_custom = semi_custom + function source_terms_custom_semi(u, x, t, equations) + t = semi_custom.t[] + return -initial_condition(x, t, equations) + end +end + +# We also create a custom ODE RHS to update the current global time +# stored in the custom semidiscretization. We unpack the standard +# semidiscretization created by Trixi.jl and pass it to `Trixi.rhs!`. + +function rhs_semi_custom!(du_ode, u_ode, semi_custom, t) + semi_custom.t[] = t + Trixi.rhs!(du_ode, u_ode, semi_custom.semi, t) +end + +# Finally, we set up an `ODEProblem` and solve it numerically. + +ode_semi_custom = ODEProblem(rhs_semi_custom!, + ode.u0, + ode.tspan, + semi_custom) +sol_semi_custom = solve(ode_semi_custom, RDPK3SpFSAL49(); + ode_default_options()...) + +# If we want to make use of additional functionality provided by +# Trixi.jl, e.g., for plotting, we need to implement a few additional +# specializations. In this case, we forward everything to the standard +# semidiscretization provided by Trixi.jl wrapped in our custom +# semidiscretization. + +Base.ndims(semi::CustomSemidiscretization) = ndims(semi.semi) +function Trixi.mesh_equations_solver_cache(semi::CustomSemidiscretization) + Trixi.mesh_equations_solver_cache(semi.semi) +end + +# Now, we can plot the numerical solution as usual. + +plot(sol_semi_custom; label = "numerical sol.") +let + x = range(-1.0, 1.0; length = 200) + plot!(x, first.(initial_condition.(x, sol_semi_custom.t[end], equations)), + label = "analytical sol.", linestyle = :dash, legend = :topleft) +end +plot!(sol_semi_custom.u[1], semi, label = "u0", linestyle = :dot, legend = :topleft) + +# This also works with many callbacks as usual. However, the +# [`AnalysisCallback`](@ref) requires some special handling since it +# makes use of a performance counter contained in the standard +# semidiscretizations of Trixi.jl to report some +# [performance metrics](@ref performance-metrics). +# Here, we forward all accesses to the performance counter to the +# wrapped semidiscretization. + +function Base.getproperty(semi::CustomSemidiscretization, s::Symbol) + if s === :performance_counter + wrapped_semi = getfield(semi, :semi) + wrapped_semi.performance_counter + else + getfield(semi, s) + end +end + +# Moreover, the [`AnalysisCallback`](@ref) also performs some error +# calculations. We also need to forward them to the wrapped +# semidiscretization. + +function Trixi.calc_error_norms(func, u, t, analyzer, + semi::CustomSemidiscretization, + cache_analysis) + Trixi.calc_error_norms(func, u, t, analyzer, + semi.semi, + cache_analysis) +end + +# Now, we can work with the callbacks used before as usual. + +summary_callback = SummaryCallback() +analysis_interval = 100 +analysis_callback = AnalysisCallback(semi_custom; + interval = analysis_interval) +alive_callback = AliveCallback(; analysis_interval) +callbacks = CallbackSet(summary_callback, + analysis_callback, + alive_callback) + +sol = solve(ode_semi_custom, RDPK3SpFSAL49(); + ode_default_options()..., callback = callbacks) +summary_callback() + +# For even more advanced usage of custom semidiscretizations, you +# may look at the source code of the ones contained in Trixi.jl, e.g., +# - [`SemidiscretizationHyperbolicParabolic`](@ref) +# - [`SemidiscretizationEulerGravity`](@ref) +# - [`SemidiscretizationEulerAcoustics`](@ref) +# - [`SemidiscretizationCoupled`](@ref) + + +# ## Package versions + +# These results were obtained using the following versions. + +using InteractiveUtils +versioninfo() + +using Pkg +Pkg.status(["Trixi", "OrdinaryDiffEq", "Plots"], + mode=PKGMODE_MANIFEST) diff --git a/docs/literate/src/files/index.jl b/docs/literate/src/files/index.jl index 0c8de66bf42..d42695611f6 100644 --- a/docs/literate/src/files/index.jl +++ b/docs/literate/src/files/index.jl @@ -76,21 +76,30 @@ # In this part, another physics model is implemented, the nonconservative linear advection equation. # We run two different simulations with different levels of refinement and compare the resulting errors. -# ### [10 Adaptive mesh refinement](@ref adaptive_mesh_refinement) +# ### [10 Parabolic terms](@ref parabolic_terms) +#- +# This tutorial describes how parabolic terms are implemented in Trixi.jl, e.g., +# to solve the advection-diffusion equation. + +# ### [11 Adding new parabolic terms](@ref adding_new_parabolic_terms) +#- +# This tutorial describes how new parabolic terms can be implemented using Trixi.jl. + +# ### [12 Adaptive mesh refinement](@ref adaptive_mesh_refinement) #- # Adaptive mesh refinement (AMR) helps to increase the accuracy in sensitive or turbolent regions while # not wasting resources for less interesting parts of the domain. This leads to much more efficient # simulations. This tutorial presents the implementation strategy of AMR in Trixi.jl, including the use of # different indicators and controllers. -# ### [11 Structured mesh with curvilinear mapping](@ref structured_mesh_mapping) +# ### [13 Structured mesh with curvilinear mapping](@ref structured_mesh_mapping) #- # In this tutorial, the use of Trixi.jl's structured curved mesh type [`StructuredMesh`](@ref) is explained. # We present the two basic option to initialize such a mesh. First, the curved domain boundaries # of a circular cylinder are set by explicit boundary functions. Then, a fully curved mesh is # defined by passing the transformation mapping. -# ### [12 Unstructured meshes with HOHQMesh.jl](@ref hohqmesh_tutorial) +# ### [14 Unstructured meshes with HOHQMesh.jl](@ref hohqmesh_tutorial) #- # The purpose of this tutorial is to demonstrate how to use the [`UnstructuredMesh2D`](@ref) # functionality of Trixi.jl. This begins by running and visualizing an available unstructured @@ -99,19 +108,24 @@ # software in the Trixi.jl ecosystem, and then run a simulation using Trixi.jl on said mesh. # In the end, the tutorial briefly explains how to simulate an example using AMR via `P4estMesh`. -# ### [13 Explicit time stepping](@ref time_stepping) +# ### [15 Explicit time stepping](@ref time_stepping) #- # This tutorial is about time integration using [OrdinaryDiffEq.jl](https://github.com/SciML/OrdinaryDiffEq.jl). # It explains how to use their algorithms and presents two types of time step choices - with error-based # and CFL-based adaptive step size control. -# ### [14 Differentiable programming](@ref differentiable_programming) +# ### [16 Differentiable programming](@ref differentiable_programming) #- # This part deals with some basic differentiable programming topics. For example, a Jacobian, its # eigenvalues and a curve of total energy (through the simulation) are calculated and plotted for # a few semidiscretizations. Moreover, we calculate an example for propagating errors with Measurement.jl # at the end. +# ### [17 Custom semidiscretization](@ref custom_semidiscretization) +#- +# This tutorial describes the [semidiscretiations](@ref overview-semidiscretizations) of Trixi.jl +# and explains how to extend them for custom tasks. + # ## Examples in Trixi.jl diff --git a/docs/make.jl b/docs/make.jl index 57629577ddb..f882fcf1219 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -68,6 +68,7 @@ files = [ # Topic: other stuff "Explicit time stepping" => "time_stepping.jl", "Differentiable programming" => "differentiable_programming.jl", + "Custom semidiscretizations" => "custom_semidiscretization.jl" ] tutorials = create_tutorials(files) diff --git a/docs/src/callbacks.md b/docs/src/callbacks.md index 1d3e5e34b51..7f44dfd5925 100644 --- a/docs/src/callbacks.md +++ b/docs/src/callbacks.md @@ -30,7 +30,7 @@ An example elixir using AMR can be found at [`examples/tree_2d_dgsem/elixir_adve The [`AnalysisCallback`](@ref) can be used to analyze the numerical solution, e.g. calculate errors or user-specified integrals, and print the results to the screen. The results can also be saved in a file. An example can be found at [`examples/tree_2d_dgsem/elixir_euler_vortex.jl`](https://github.com/trixi-framework/Trixi.jl/blob/main/examples/tree_2d_dgsem/elixir_euler_vortex.jl). -In [Performance metrics of the `AnalysisCallback`](@ref) you can find a detailed +In [Performance metrics of the `AnalysisCallback`](@ref performance-metrics) you can find a detailed description of the different performance metrics the `AnalysisCallback` computes. ### I/O @@ -106,7 +106,7 @@ will yield the following plot: the automated performance measurements, including an output of the recorded timers after a simulation. * The [`VisualizationCallback`](@ref) can be used for in-situ visualization. See [Visualizing results during a simulation](@ref). -* The [`TrivialCallback`](@ref) does nothing and can be used to to easily disable some callbacks +* The [`TrivialCallback`](@ref) does nothing and can be used to easily disable some callbacks via [`trixi_include`](@ref). ### Equation-specific callbacks diff --git a/docs/src/overview.md b/docs/src/overview.md index 46bc28b6025..51a6272ae8e 100644 --- a/docs/src/overview.md +++ b/docs/src/overview.md @@ -16,7 +16,7 @@ to solve a PDE numerically are the spatial semidiscretization and the time integration scheme. -## Semidiscretizations +## [Semidiscretizations](@id overview-semidiscretizations) Semidiscretizations are high-level descriptions of spatial discretizations specialized for certain PDEs. Trixi.jl's main focus is on hyperbolic conservation diff --git a/docs/src/parallelization.md b/docs/src/parallelization.md index d56777c9af4..e55471bb256 100644 --- a/docs/src/parallelization.md +++ b/docs/src/parallelization.md @@ -22,7 +22,7 @@ julia --threads=4 If both the environment variable and the command line argument are specified at the same time, the latter takes precedence. -If you use time integration methods from +If you use time integration methods from [OrdinaryDiffEq.jl](https://github.com/SciML/OrdinaryDiffEq.jl) and want to use multiple threads therein, you need to set the keyword argument `thread=OrdinaryDiffEq.True()` of the algorithms, as described in the @@ -143,7 +143,7 @@ To start Trixi.jl in parallel with MPI, there are three options: Switching between panes can be done by `Ctrl+b` followed by `o`. As of March 2022, newer versions of tmpi also support mpich, which is the default backend of MPI.jl (via MPICH_Jll.jl). To use this setup, you need to install - `mpiexecjl` as described in the + `mpiexecjl` as described in the [documentation of MPI.jl](https://juliaparallel.org/MPI.jl/v0.20/usage/#Julia-wrapper-for-mpiexec) and make it available as `mpirun`, e.g., via a symlink of the form ```bash @@ -161,7 +161,7 @@ To start Trixi.jl in parallel with MPI, there are three options: ### [Performance](@id parallel_performance) For information on how to evaluate the parallel performance of Trixi.jl, please -have a look at the [Performance metrics of the `AnalysisCallback`](@ref) +have a look at the [Performance metrics of the `AnalysisCallback`](@ref performance-metrics) section, specifically at the descriptions of the performance index (PID). diff --git a/docs/src/performance.md b/docs/src/performance.md index 428672ec75f..bbe3a3390b7 100644 --- a/docs/src/performance.md +++ b/docs/src/performance.md @@ -170,7 +170,7 @@ As a rule of thumb: - Consider using `@nospecialize` for methods like custom implementations of `Base.show`. -## Performance metrics of the `AnalysisCallback` +## [Performance metrics of the `AnalysisCallback`](@id performance-metrics) The [`AnalysisCallback`](@ref) computes two performance indicators that you can use to evaluate the serial and parallel performance of Trixi.jl. They represent measured run times that are normalized by the number of `rhs!` evaluations and diff --git a/docs/src/restart.md b/docs/src/restart.md index 767269ff27d..c7cbcd11852 100644 --- a/docs/src/restart.md +++ b/docs/src/restart.md @@ -77,8 +77,7 @@ and its time step number, e.g.: ```julia integrator = init(ode, CarpenterKennedy2N54(williamson_condition=false), dt=dt, save_everystep=false, callback=callbacks); -integrator.iter = load_timestep(restart_filename) -integrator.stats.naccept = integrator.iter +load_timestep!(integrator, restart_filename) ``` Now we can compute the solution: diff --git a/examples/p4est_2d_dgsem/elixir_advection_restart.jl b/examples/p4est_2d_dgsem/elixir_advection_restart.jl index 79a35199b83..52917616a6a 100644 --- a/examples/p4est_2d_dgsem/elixir_advection_restart.jl +++ b/examples/p4est_2d_dgsem/elixir_advection_restart.jl @@ -35,8 +35,7 @@ integrator = init(ode, CarpenterKennedy2N54(williamson_condition=false), save_everystep=false, callback=callbacks); # Get the last time index and work with that. -integrator.iter = load_timestep(restart_filename) -integrator.stats.naccept = integrator.iter +load_timestep!(integrator, restart_filename) ############################################################################### diff --git a/examples/p4est_2d_dgsem/elixir_navierstokes_convergence.jl b/examples/p4est_2d_dgsem/elixir_navierstokes_convergence.jl index 8111df8251a..b0c6086ad63 100644 --- a/examples/p4est_2d_dgsem/elixir_navierstokes_convergence.jl +++ b/examples/p4est_2d_dgsem/elixir_navierstokes_convergence.jl @@ -170,7 +170,10 @@ end initial_condition = initial_condition_navier_stokes_convergence_test # BC types -velocity_bc_top_bottom = NoSlip((x, t, equations) -> initial_condition_navier_stokes_convergence_test(x, t, equations)[2:3]) +velocity_bc_top_bottom = NoSlip() do x, t, equations + u = initial_condition_navier_stokes_convergence_test(x, t, equations) + return SVector(u[2], u[3]) +end heat_bc_top_bottom = Adiabatic((x, t, equations) -> 0.0) boundary_condition_top_bottom = BoundaryConditionNavierStokesWall(velocity_bc_top_bottom, heat_bc_top_bottom) diff --git a/examples/p4est_3d_dgsem/elixir_advection_restart.jl b/examples/p4est_3d_dgsem/elixir_advection_restart.jl index b27eaab62e2..26d10cf8826 100644 --- a/examples/p4est_3d_dgsem/elixir_advection_restart.jl +++ b/examples/p4est_3d_dgsem/elixir_advection_restart.jl @@ -32,8 +32,7 @@ integrator = init(ode, CarpenterKennedy2N54(williamson_condition=false), save_everystep=false, callback=callbacks); # Get the last time index and work with that. -integrator.iter = load_timestep(restart_filename) -integrator.stats.naccept = integrator.iter +load_timestep!(integrator, restart_filename) ############################################################################### diff --git a/examples/p4est_3d_dgsem/elixir_navierstokes_convergence.jl b/examples/p4est_3d_dgsem/elixir_navierstokes_convergence.jl index c426fe95f5b..0109e58dfb3 100644 --- a/examples/p4est_3d_dgsem/elixir_navierstokes_convergence.jl +++ b/examples/p4est_3d_dgsem/elixir_navierstokes_convergence.jl @@ -220,7 +220,10 @@ end initial_condition = initial_condition_navier_stokes_convergence_test # BC types -velocity_bc_top_bottom = NoSlip((x, t, equations) -> initial_condition_navier_stokes_convergence_test(x, t, equations)[2:4]) +velocity_bc_top_bottom = NoSlip() do x, t, equations + u = initial_condition_navier_stokes_convergence_test(x, t, equations) + return SVector(u[2], u[3], u[4]) +end heat_bc_top_bottom = Adiabatic((x, t, equations) -> 0.0) boundary_condition_top_bottom = BoundaryConditionNavierStokesWall(velocity_bc_top_bottom, heat_bc_top_bottom) diff --git a/examples/structured_2d_dgsem/elixir_advection_restart.jl b/examples/structured_2d_dgsem/elixir_advection_restart.jl index 98c44fac71a..82eaa21333a 100644 --- a/examples/structured_2d_dgsem/elixir_advection_restart.jl +++ b/examples/structured_2d_dgsem/elixir_advection_restart.jl @@ -34,8 +34,7 @@ integrator = init(ode, CarpenterKennedy2N54(williamson_condition=false), save_everystep=false, callback=callbacks); # Get the last time index and work with that. -integrator.iter = load_timestep(restart_filename) -integrator.stats.naccept = integrator.iter +load_timestep!(integrator, restart_filename) ############################################################################### # run the simulation diff --git a/examples/structured_3d_dgsem/elixir_advection_restart.jl b/examples/structured_3d_dgsem/elixir_advection_restart.jl index 39d28848c77..921c5310340 100644 --- a/examples/structured_3d_dgsem/elixir_advection_restart.jl +++ b/examples/structured_3d_dgsem/elixir_advection_restart.jl @@ -32,8 +32,7 @@ integrator = init(ode, CarpenterKennedy2N54(williamson_condition=false), save_everystep=false, callback=callbacks); # Get the last time index and work with that. -integrator.iter = load_timestep(restart_filename) -integrator.stats.naccept = integrator.iter +load_timestep!(integrator, restart_filename) ############################################################################### diff --git a/examples/tree_1d_dgsem/elixir_navierstokes_convergence_walls_amr.jl b/examples/tree_1d_dgsem/elixir_navierstokes_convergence_walls_amr.jl new file mode 100644 index 00000000000..1daeab04a71 --- /dev/null +++ b/examples/tree_1d_dgsem/elixir_navierstokes_convergence_walls_amr.jl @@ -0,0 +1,172 @@ +using OrdinaryDiffEq +using Trixi + +############################################################################### +# semidiscretization of the ideal compressible Navier-Stokes equations + +prandtl_number() = 0.72 +mu() = 0.01 + +equations = CompressibleEulerEquations1D(1.4) +equations_parabolic = CompressibleNavierStokesDiffusion1D(equations, mu=mu(), Prandtl=prandtl_number(), + gradient_variables=GradientVariablesEntropy()) + +# Create DG solver with polynomial degree = 3 and (local) Lax-Friedrichs/Rusanov flux as surface flux +solver = DGSEM(polydeg=3, surface_flux=flux_lax_friedrichs, + volume_integral=VolumeIntegralWeakForm()) + +coordinates_min = -1.0 +coordinates_max = 1.0 + +# Create a uniformly refined mesh with periodic boundaries +mesh = TreeMesh(coordinates_min, coordinates_max, + initial_refinement_level=3, + periodicity=false, + n_cells_max=30_000) # set maximum capacity of tree data structure + +# Note: the initial condition cannot be specialized to `CompressibleNavierStokesDiffusion1D` +# since it is called by both the parabolic solver (which passes in `CompressibleNavierStokesDiffusion1D`) +# and by the initial condition (which passes in `CompressibleEulerEquations1D`). +# This convergence test setup was originally derived by Andrew Winters (@andrewwinters5000) +function initial_condition_navier_stokes_convergence_test(x, t, equations) + # Amplitude and shift + A = 0.5 + c = 2.0 + + # convenience values for trig. functions + pi_x = pi * x[1] + pi_t = pi * t + + rho = c + A * cos(pi_x) * cos(pi_t) + v1 = log(x[1] + 2.0) * (1.0 - exp(-A * (x[1] - 1.0)) ) * cos(pi_t) + p = rho^2 + + return prim2cons(SVector(rho, v1, p), equations) +end + +@inline function source_terms_navier_stokes_convergence_test(u, x, t, equations) + x = x[1] + + # TODO: parabolic + # we currently need to hardcode these parameters until we fix the "combined equation" issue + # see also https://github.com/trixi-framework/Trixi.jl/pull/1160 + inv_gamma_minus_one = inv(equations.gamma - 1) + Pr = prandtl_number() + mu_ = mu() + + # Same settings as in `initial_condition` + # Amplitude and shift + A = 0.5 + c = 2.0 + + # convenience values for trig. functions + pi_x = pi * x + pi_t = pi * t + + # compute the manufactured solution and all necessary derivatives + rho = c + A * cos(pi_x) * cos(pi_t) + rho_t = -pi * A * cos(pi_x) * sin(pi_t) + rho_x = -pi * A * sin(pi_x) * cos(pi_t) + rho_xx = -pi * pi * A * cos(pi_x) * cos(pi_t) + + v1 = log(x + 2.0) * (1.0 - exp(-A * (x - 1.0))) * cos(pi_t) + v1_t = -pi * log(x + 2.0) * (1.0 - exp(-A * (x - 1.0))) * sin(pi_t) + v1_x = (A * log(x + 2.0) * exp(-A * (x - 1.0)) + (1.0 - exp(-A * (x - 1.0))) / (x + 2.0)) * cos(pi_t) + v1_xx = (( 2.0 * A * exp(-A * (x - 1.0)) / (x + 2.0) + - A * A * log(x + 2.0) * exp(-A * (x - 1.0)) + - (1.0 - exp(-A * (x - 1.0))) / ((x + 2.0) * (x + 2.0))) * cos(pi_t)) + + p = rho * rho + p_t = 2.0 * rho * rho_t + p_x = 2.0 * rho * rho_x + p_xx = 2.0 * rho * rho_xx + 2.0 * rho_x * rho_x + + # Note this simplifies slightly because the ansatz assumes that v1 = v2 + E = p * inv_gamma_minus_one + 0.5 * rho * v1^2 + E_t = p_t * inv_gamma_minus_one + 0.5 * rho_t * v1^2 + rho * v1 * v1_t + E_x = p_x * inv_gamma_minus_one + 0.5 * rho_x * v1^2 + rho * v1 * v1_x + + # Some convenience constants + T_const = equations.gamma * inv_gamma_minus_one / Pr + inv_rho_cubed = 1.0 / (rho^3) + + # compute the source terms + # density equation + du1 = rho_t + rho_x * v1 + rho * v1_x + + # y-momentum equation + du2 = ( rho_t * v1 + rho * v1_t + + p_x + rho_x * v1^2 + 2.0 * rho * v1 * v1_x + # stress tensor from y-direction + - v1_xx * mu_) + + # total energy equation + du3 = ( E_t + v1_x * (E + p) + v1 * (E_x + p_x) + # stress tensor and temperature gradient terms from x-direction + - v1_xx * v1 * mu_ + - v1_x * v1_x * mu_ + - T_const * inv_rho_cubed * ( p_xx * rho * rho + - 2.0 * p_x * rho * rho_x + + 2.0 * p * rho_x * rho_x + - p * rho * rho_xx ) * mu_ ) + + return SVector(du1, du2, du3) +end + +initial_condition = initial_condition_navier_stokes_convergence_test + +# BC types +velocity_bc_left_right = NoSlip((x, t, equations) -> initial_condition_navier_stokes_convergence_test(x, t, equations)[2]) + +heat_bc_left = Isothermal((x, t, equations) -> + Trixi.temperature(initial_condition_navier_stokes_convergence_test(x, t, equations), + equations_parabolic)) +heat_bc_right = Adiabatic((x, t, equations) -> 0.0) + +boundary_condition_left = BoundaryConditionNavierStokesWall(velocity_bc_left_right, heat_bc_left) +boundary_condition_right = BoundaryConditionNavierStokesWall(velocity_bc_left_right, heat_bc_right) + +# define inviscid boundary conditions +boundary_conditions = (; x_neg = boundary_condition_slip_wall, + x_pos = boundary_condition_slip_wall) + +# define viscous boundary conditions +boundary_conditions_parabolic = (; x_neg = boundary_condition_left, + x_pos = boundary_condition_right) + +semi = SemidiscretizationHyperbolicParabolic(mesh, (equations, equations_parabolic), initial_condition, solver; + boundary_conditions=(boundary_conditions, boundary_conditions_parabolic), + source_terms=source_terms_navier_stokes_convergence_test) + +############################################################################### +# ODE solvers, callbacks etc. + +# Create ODE problem with time span `tspan` +tspan = (0.0, 1.0) +ode = semidiscretize(semi, tspan) + +summary_callback = SummaryCallback() +alive_callback = AliveCallback(alive_interval=10) +analysis_interval = 100 +analysis_callback = AnalysisCallback(semi, interval=analysis_interval) + +amr_controller = ControllerThreeLevel(semi, + IndicatorLöhner(semi, variable=Trixi.density), + base_level=3, + med_level=4, med_threshold=0.005, + max_level=5, max_threshold=0.01) + +amr_callback = AMRCallback(semi, amr_controller, + interval=5, + adapt_initial_condition=true) + +# Create a CallbackSet to collect all callbacks such that they can be passed to the ODE solver +callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, amr_callback) + +############################################################################### +# run the simulation + +time_int_tol = 1e-8 +sol = solve(ode, RDPK3SpFSAL49(); abstol=time_int_tol, reltol=time_int_tol, dt = 1e-5, + ode_default_options()..., callback=callbacks) +summary_callback() # print the timer summary \ No newline at end of file diff --git a/examples/tree_2d_dgsem/elixir_advection_restart.jl b/examples/tree_2d_dgsem/elixir_advection_restart.jl index 72efb7d0c84..771ec5aefe7 100644 --- a/examples/tree_2d_dgsem/elixir_advection_restart.jl +++ b/examples/tree_2d_dgsem/elixir_advection_restart.jl @@ -32,8 +32,7 @@ integrator = init(ode, alg, save_everystep=false, callback=callbacks) # Get the last time index and work with that. -integrator.iter = load_timestep(restart_filename) -integrator.stats.naccept = integrator.iter +load_timestep!(integrator, restart_filename) ############################################################################### # run the simulation diff --git a/examples/tree_3d_dgsem/elixir_advection_restart.jl b/examples/tree_3d_dgsem/elixir_advection_restart.jl index 3061f165874..b7835ed061f 100644 --- a/examples/tree_3d_dgsem/elixir_advection_restart.jl +++ b/examples/tree_3d_dgsem/elixir_advection_restart.jl @@ -31,8 +31,7 @@ integrator = init(ode, CarpenterKennedy2N54(williamson_condition=false), save_everystep=false, callback=callbacks); # Get the last time index and work with that. -integrator.iter = load_timestep(restart_filename) -integrator.stats.naccept = integrator.iter +load_timestep!(integrator, restart_filename) ############################################################################### diff --git a/examples/unstructured_2d_dgsem/elixir_euler_restart.jl b/examples/unstructured_2d_dgsem/elixir_euler_restart.jl index b85cc2c6d70..6653f8662d9 100644 --- a/examples/unstructured_2d_dgsem/elixir_euler_restart.jl +++ b/examples/unstructured_2d_dgsem/elixir_euler_restart.jl @@ -33,8 +33,7 @@ integrator = init(ode, CarpenterKennedy2N54(williamson_condition=false), save_everystep=false, callback=callbacks); # Get the last time index and work with that. -integrator.iter = load_timestep(restart_filename) -integrator.stats.naccept = integrator.iter +load_timestep!(integrator, restart_filename) ############################################################################### diff --git a/src/Trixi.jl b/src/Trixi.jl index e171d90c924..c883c3bf19f 100644 --- a/src/Trixi.jl +++ b/src/Trixi.jl @@ -255,7 +255,7 @@ export SummaryCallback, SteadyStateCallback, AnalysisCallback, AliveCallback, GlmSpeedCallback, LBMCollisionCallback, EulerAcousticsCouplingCallback, TrivialCallback, AnalysisCallbackCoupled -export load_mesh, load_time, load_timestep, load_dt +export load_mesh, load_time, load_timestep, load_timestep!, load_dt export ControllerThreeLevel, ControllerThreeLevelCombined, IndicatorLöhner, IndicatorLoehner, IndicatorMax, diff --git a/src/callbacks_step/amr.jl b/src/callbacks_step/amr.jl index 4d80e6e1139..ba840ff9675 100644 --- a/src/callbacks_step/amr.jl +++ b/src/callbacks_step/amr.jl @@ -192,6 +192,16 @@ end amr_callback(u_ode, mesh_equations_solver_cache(semi)..., semi, t, iter; kwargs...) end +@inline function (amr_callback::AMRCallback)(u_ode::AbstractVector, + semi::SemidiscretizationHyperbolicParabolic, + t, iter; + kwargs...) + # Note that we don't `wrap_array` the vector `u_ode` to be able to `resize!` + # it when doing AMR while still dispatching on the `mesh` etc. + amr_callback(u_ode, mesh_equations_solver_cache(semi)..., semi.cache_parabolic, + semi, t, iter; kwargs...) +end + # `passive_args` is currently used for Euler with self-gravity to adapt the gravity solver # passively without querying its indicator, based on the assumption that both solvers use # the same mesh. That's a hack and should be improved in the future once we have more examples @@ -346,6 +356,154 @@ function (amr_callback::AMRCallback)(u_ode::AbstractVector, mesh::TreeMesh, return has_changed end +function (amr_callback::AMRCallback)(u_ode::AbstractVector, mesh::TreeMesh, + equations, dg::DG, + cache, cache_parabolic, + semi::SemidiscretizationHyperbolicParabolic, + t, iter; + only_refine = false, only_coarsen = false) + @unpack controller, adaptor = amr_callback + + u = wrap_array(u_ode, mesh, equations, dg, cache) + # Indicator kept based on hyperbolic variables + lambda = @trixi_timeit timer() "indicator" controller(u, mesh, equations, dg, cache, + t = t, iter = iter) + + if mpi_isparallel() + error("MPI has not been verified yet for parabolic AMR") + + # Collect lambda for all elements + lambda_global = Vector{eltype(lambda)}(undef, nelementsglobal(dg, cache)) + # Use parent because n_elements_by_rank is an OffsetArray + recvbuf = MPI.VBuffer(lambda_global, parent(cache.mpi_cache.n_elements_by_rank)) + MPI.Allgatherv!(lambda, recvbuf, mpi_comm()) + lambda = lambda_global + end + + leaf_cell_ids = leaf_cells(mesh.tree) + @boundscheck begin + @assert axes(lambda)==axes(leaf_cell_ids) ("Indicator (axes = $(axes(lambda))) and leaf cell (axes = $(axes(leaf_cell_ids))) arrays have different axes") + end + + @unpack to_refine, to_coarsen = amr_callback.amr_cache + empty!(to_refine) + empty!(to_coarsen) + for element in 1:length(lambda) + controller_value = lambda[element] + if controller_value > 0 + push!(to_refine, leaf_cell_ids[element]) + elseif controller_value < 0 + push!(to_coarsen, leaf_cell_ids[element]) + end + end + + @trixi_timeit timer() "refine" if !only_coarsen && !isempty(to_refine) + # refine mesh + refined_original_cells = @trixi_timeit timer() "mesh" refine!(mesh.tree, + to_refine) + + # Find all indices of elements whose cell ids are in refined_original_cells + # Note: This assumes same indices for hyperbolic and parabolic part. + elements_to_refine = findall(in(refined_original_cells), + cache.elements.cell_ids) + + # refine solver + @trixi_timeit timer() "solver" refine!(u_ode, adaptor, mesh, equations, dg, + cache, cache_parabolic, + elements_to_refine) + else + # If there is nothing to refine, create empty array for later use + refined_original_cells = Int[] + end + + @trixi_timeit timer() "coarsen" if !only_refine && !isempty(to_coarsen) + # Since the cells may have been shifted due to refinement, first we need to + # translate the old cell ids to the new cell ids + if !isempty(to_coarsen) + to_coarsen = original2refined(to_coarsen, refined_original_cells, mesh) + end + + # Next, determine the parent cells from which the fine cells are to be + # removed, since these are needed for the coarsen! function. However, since + # we only want to coarsen if *all* child cells are marked for coarsening, + # we count the coarsening indicators for each parent cell and only coarsen + # if all children are marked as such (i.e., where the count is 2^ndims). At + # the same time, check if a cell is marked for coarsening even though it is + # *not* a leaf cell -> this can only happen if it was refined due to 2:1 + # smoothing during the preceding refinement operation. + parents_to_coarsen = zeros(Int, length(mesh.tree)) + for cell_id in to_coarsen + # If cell has no parent, it cannot be coarsened + if !has_parent(mesh.tree, cell_id) + continue + end + + # If cell is not leaf (anymore), it cannot be coarsened + if !is_leaf(mesh.tree, cell_id) + continue + end + + # Increase count for parent cell + parent_id = mesh.tree.parent_ids[cell_id] + parents_to_coarsen[parent_id] += 1 + end + + # Extract only those parent cells for which all children should be coarsened + to_coarsen = collect(1:length(parents_to_coarsen))[parents_to_coarsen .== 2^ndims(mesh)] + + # Finally, coarsen mesh + coarsened_original_cells = @trixi_timeit timer() "mesh" coarsen!(mesh.tree, + to_coarsen) + + # Convert coarsened parent cell ids to the list of child cell ids that have + # been removed, since this is the information that is expected by the solver + removed_child_cells = zeros(Int, + n_children_per_cell(mesh.tree) * + length(coarsened_original_cells)) + for (index, coarse_cell_id) in enumerate(coarsened_original_cells) + for child in 1:n_children_per_cell(mesh.tree) + removed_child_cells[n_children_per_cell(mesh.tree) * (index - 1) + child] = coarse_cell_id + + child + end + end + + # Find all indices of elements whose cell ids are in removed_child_cells + # Note: This assumes same indices for hyperbolic and parabolic part. + elements_to_remove = findall(in(removed_child_cells), cache.elements.cell_ids) + + # coarsen solver + @trixi_timeit timer() "solver" coarsen!(u_ode, adaptor, mesh, equations, dg, + cache, cache_parabolic, + elements_to_remove) + else + # If there is nothing to coarsen, create empty array for later use + coarsened_original_cells = Int[] + end + + # Store whether there were any cells coarsened or refined + has_changed = !isempty(refined_original_cells) || !isempty(coarsened_original_cells) + if has_changed # TODO: Taal decide, where shall we set this? + # don't set it to has_changed since there can be changes from earlier calls + mesh.unsaved_changes = true + end + + # Dynamically balance computational load by first repartitioning the mesh and then redistributing the cells/elements + if has_changed && mpi_isparallel() && amr_callback.dynamic_load_balancing + error("MPI has not been verified yet for parabolic AMR") + + @trixi_timeit timer() "dynamic load balancing" begin + old_mpi_ranks_per_cell = copy(mesh.tree.mpi_ranks) + + partition!(mesh) + + rebalance_solver!(u_ode, mesh, equations, dg, cache, old_mpi_ranks_per_cell) + end + end + + # Return true if there were any cells coarsened or refined, otherwise false + return has_changed +end + # Copy controller values to quad user data storage, will be called below function copy_to_quad_iter_volume(info, user_data) info_pw = PointerWrapper(info) diff --git a/src/callbacks_step/amr_dg1d.jl b/src/callbacks_step/amr_dg1d.jl index e31a74730ea..e721ccc61cb 100644 --- a/src/callbacks_step/amr_dg1d.jl +++ b/src/callbacks_step/amr_dg1d.jl @@ -76,6 +76,44 @@ function refine!(u_ode::AbstractVector, adaptor, mesh::TreeMesh{1}, return nothing end +function refine!(u_ode::AbstractVector, adaptor, mesh::TreeMesh{1}, + equations, dg::DGSEM, cache, cache_parabolic, + elements_to_refine) + # Call `refine!` for the hyperbolic part, which does the heavy lifting of + # actually transferring the solution to the refined cells + refine!(u_ode, adaptor, mesh, equations, dg, cache, elements_to_refine) + + # The remaining function only handles the necessary adaptation of the data structures + # for the parabolic part of the semidiscretization + + # Get new list of leaf cells + leaf_cell_ids = local_leaf_cells(mesh.tree) + + @unpack elements, viscous_container = cache_parabolic + resize!(elements, length(leaf_cell_ids)) + init_elements!(elements, leaf_cell_ids, mesh, dg.basis) + + # Resize parabolic helper variables + resize!(viscous_container, equations, dg, cache) + + # re-initialize interfaces container + @unpack interfaces = cache_parabolic + resize!(interfaces, count_required_interfaces(mesh, leaf_cell_ids)) + init_interfaces!(interfaces, elements, mesh) + + # re-initialize boundaries container + @unpack boundaries = cache_parabolic + resize!(boundaries, count_required_boundaries(mesh, leaf_cell_ids)) + init_boundaries!(boundaries, elements, mesh) + + # Sanity check + if isperiodic(mesh.tree) + @assert ninterfaces(interfaces)==1 * nelements(dg, cache_parabolic) ("For 1D and periodic domains, the number of interfaces must be the same as the number of elements") + end + + return nothing +end + # TODO: Taal compare performance of different implementations # Refine solution data u for an element, using L2 projection (interpolation) function refine_element!(u::AbstractArray{<:Any, 3}, element_id, @@ -201,6 +239,41 @@ function coarsen!(u_ode::AbstractVector, adaptor, mesh::TreeMesh{1}, return nothing end +function coarsen!(u_ode::AbstractVector, adaptor, mesh::TreeMesh{1}, + equations, dg::DGSEM, cache, cache_parabolic, + elements_to_remove) + # Call `coarsen!` for the hyperbolic part, which does the heavy lifting of + # actually transferring the solution to the coarsened cells + coarsen!(u_ode, adaptor, mesh, equations, dg, cache, elements_to_remove) + + # Get new list of leaf cells + leaf_cell_ids = local_leaf_cells(mesh.tree) + + @unpack elements, viscous_container = cache_parabolic + resize!(elements, length(leaf_cell_ids)) + init_elements!(elements, leaf_cell_ids, mesh, dg.basis) + + # Resize parabolic helper variables + resize!(viscous_container, equations, dg, cache) + + # re-initialize interfaces container + @unpack interfaces = cache_parabolic + resize!(interfaces, count_required_interfaces(mesh, leaf_cell_ids)) + init_interfaces!(interfaces, elements, mesh) + + # re-initialize boundaries container + @unpack boundaries = cache_parabolic + resize!(boundaries, count_required_boundaries(mesh, leaf_cell_ids)) + init_boundaries!(boundaries, elements, mesh) + + # Sanity check + if isperiodic(mesh.tree) + @assert ninterfaces(interfaces)==1 * nelements(dg, cache_parabolic) ("For 1D and periodic domains, the number of interfaces must be the same as the number of elements") + end + + return nothing +end + # TODO: Taal compare performance of different implementations # Coarsen solution data u for two elements, using L2 projection function coarsen_elements!(u::AbstractArray{<:Any, 3}, element_id, diff --git a/src/callbacks_step/save_restart.jl b/src/callbacks_step/save_restart.jl index f567a5c7fda..06817a9b730 100644 --- a/src/callbacks_step/save_restart.jl +++ b/src/callbacks_step/save_restart.jl @@ -141,6 +141,18 @@ function load_timestep(restart_file::AbstractString) end end +""" + load_timestep!(integrator, restart_file::AbstractString) + +Load the time step number saved in a `restart_file` and assign it to both the time step +number and and the number of accepted steps +(`iter` and `stats.naccept` in OrdinaryDiffEq.jl, respectively) in `integrator`. +""" +function load_timestep!(integrator, restart_file::AbstractString) + integrator.iter = load_timestep(restart_file) + integrator.stats.naccept = integrator.iter +end + """ load_dt(restart_file::AbstractString) diff --git a/src/solvers/dgmulti/dg_parabolic.jl b/src/solvers/dgmulti/dg_parabolic.jl index 72dbe2c4256..7dfe4430244 100644 --- a/src/solvers/dgmulti/dg_parabolic.jl +++ b/src/solvers/dgmulti/dg_parabolic.jl @@ -62,9 +62,10 @@ end function transform_variables!(u_transformed, u, mesh, equations_parabolic::AbstractEquationsParabolic, dg::DGMulti, parabolic_scheme, cache, cache_parabolic) + transformation = gradient_variable_transformation(equations_parabolic) + @threaded for i in eachindex(u) - u_transformed[i] = gradient_variable_transformation(equations_parabolic)(u[i], - equations_parabolic) + u_transformed[i] = transformation(u[i], equations_parabolic) end end diff --git a/src/solvers/dgsem_p4est/dg_2d.jl b/src/solvers/dgsem_p4est/dg_2d.jl index 97b931fa325..a665aa4b19d 100644 --- a/src/solvers/dgsem_p4est/dg_2d.jl +++ b/src/solvers/dgsem_p4est/dg_2d.jl @@ -275,9 +275,9 @@ function prolong2boundaries!(cache, u, return nothing end -function calc_boundary_flux!(cache, t, boundary_condition, boundary_indexing, +function calc_boundary_flux!(cache, t, boundary_condition::BC, boundary_indexing, mesh::Union{P4estMesh{2}, T8codeMesh{2}}, - equations, surface_integral, dg::DG) + equations, surface_integral, dg::DG) where {BC} @unpack boundaries = cache @unpack surface_flux_values = cache.elements index_range = eachnode(dg) diff --git a/src/solvers/dgsem_tree/container_viscous_1d.jl b/src/solvers/dgsem_tree/container_viscous_1d.jl new file mode 100644 index 00000000000..a4919f75396 --- /dev/null +++ b/src/solvers/dgsem_tree/container_viscous_1d.jl @@ -0,0 +1,58 @@ +mutable struct ViscousContainer1D{uEltype <: Real} + u_transformed::Array{uEltype, 3} + gradients::Array{uEltype, 3} + flux_viscous::Array{uEltype, 3} + + # internal `resize!`able storage + _u_transformed::Vector{uEltype} + _gradients::Vector{uEltype} + _flux_viscous::Vector{uEltype} + + function ViscousContainer1D{uEltype}(n_vars::Integer, n_nodes::Integer, + n_elements::Integer) where {uEltype <: Real} + new(Array{uEltype, 3}(undef, n_vars, n_nodes, n_elements), + Array{uEltype, 3}(undef, n_vars, n_nodes, n_elements), + Array{uEltype, 3}(undef, n_vars, n_nodes, n_elements), + Vector{uEltype}(undef, n_vars * n_nodes * n_elements), + Vector{uEltype}(undef, n_vars * n_nodes * n_elements), + Vector{uEltype}(undef, n_vars * n_nodes * n_elements)) + end +end + +function init_viscous_container(n_vars::Integer, n_nodes::Integer, + n_elements::Integer, + ::Type{uEltype}) where {uEltype <: Real} + return ViscousContainer1D{uEltype}(n_vars, n_nodes, n_elements) +end + +# Only one-dimensional `Array`s are `resize!`able in Julia. +# Hence, we use `Vector`s as internal storage and `resize!` +# them whenever needed. Then, we reuse the same memory by +# `unsafe_wrap`ping multi-dimensional `Array`s around the +# internal storage. +function Base.resize!(viscous_container::ViscousContainer1D, equations, dg, cache) + capacity = nvariables(equations) * nnodes(dg) * nelements(dg, cache) + resize!(viscous_container._u_transformed, capacity) + resize!(viscous_container._gradients, capacity) + resize!(viscous_container._flux_viscous, capacity) + + viscous_container.u_transformed = unsafe_wrap(Array, + pointer(viscous_container._u_transformed), + (nvariables(equations), + nnodes(dg), + nelements(dg, cache))) + + viscous_container.gradients = unsafe_wrap(Array, + pointer(viscous_container._gradients), + (nvariables(equations), + nnodes(dg), + nelements(dg, cache))) + + viscous_container.flux_viscous = unsafe_wrap(Array, + pointer(viscous_container._flux_viscous), + (nvariables(equations), + nnodes(dg), + nelements(dg, cache))) + + return nothing +end diff --git a/src/solvers/dgsem_tree/dg.jl b/src/solvers/dgsem_tree/dg.jl index 6e02bc1d94a..ff37bad3b3a 100644 --- a/src/solvers/dgsem_tree/dg.jl +++ b/src/solvers/dgsem_tree/dg.jl @@ -54,6 +54,9 @@ include("containers.jl") # Dimension-agnostic parallel setup include("dg_parallel.jl") +# Helper struct for parabolic AMR +include("container_viscous_1d.jl") + # 1D DG implementation include("dg_1d.jl") include("dg_1d_parabolic.jl") diff --git a/src/solvers/dgsem_tree/dg_1d_parabolic.jl b/src/solvers/dgsem_tree/dg_1d_parabolic.jl index c2aa75388c8..97e31e0e22b 100644 --- a/src/solvers/dgsem_tree/dg_1d_parabolic.jl +++ b/src/solvers/dgsem_tree/dg_1d_parabolic.jl @@ -17,7 +17,8 @@ function rhs_parabolic!(du, u, t, mesh::TreeMesh{1}, equations_parabolic::AbstractEquationsParabolic, initial_condition, boundary_conditions_parabolic, source_terms, dg::DG, parabolic_scheme, cache, cache_parabolic) - @unpack u_transformed, gradients, flux_viscous = cache_parabolic + @unpack viscous_container = cache_parabolic + @unpack u_transformed, gradients, flux_viscous = viscous_container # Convert conservative variables to a form more suitable for viscous flux calculations @trixi_timeit timer() "transform variables" begin @@ -105,12 +106,13 @@ end function transform_variables!(u_transformed, u, mesh::TreeMesh{1}, equations_parabolic::AbstractEquationsParabolic, dg::DG, parabolic_scheme, cache, cache_parabolic) + transformation = gradient_variable_transformation(equations_parabolic) + @threaded for element in eachelement(dg, cache) # Calculate volume terms in one element for i in eachnode(dg) u_node = get_node_vars(u, equations_parabolic, dg, i, element) - u_transformed_node = gradient_variable_transformation(equations_parabolic)(u_node, - equations_parabolic) + u_transformed_node = transformation(u_node, equations_parabolic) set_node_vars!(u_transformed, u_transformed_node, equations_parabolic, dg, i, element) end @@ -147,16 +149,18 @@ function prolong2interfaces!(cache_parabolic, flux_viscous, equations_parabolic::AbstractEquationsParabolic, surface_integral, dg::DG, cache) @unpack interfaces = cache_parabolic + @unpack neighbor_ids = interfaces + interfaces_u = interfaces.u @threaded for interface in eachinterface(dg, cache) - left_element = interfaces.neighbor_ids[1, interface] - right_element = interfaces.neighbor_ids[2, interface] + left_element = neighbor_ids[1, interface] + right_element = neighbor_ids[2, interface] # interface in x-direction for v in eachvariable(equations_parabolic) - # OBS! `interfaces.u` stores the interpolated *fluxes* and *not the solution*! - interfaces.u[1, v, interface] = flux_viscous[v, nnodes(dg), left_element] - interfaces.u[2, v, interface] = flux_viscous[v, 1, right_element] + # OBS! `interfaces_u` stores the interpolated *fluxes* and *not the solution*! + interfaces_u[1, v, interface] = flux_viscous[v, nnodes(dg), left_element] + interfaces_u[2, v, interface] = flux_viscous[v, 1, right_element] end end @@ -204,21 +208,22 @@ function prolong2boundaries!(cache_parabolic, flux_viscous, equations_parabolic::AbstractEquationsParabolic, surface_integral, dg::DG, cache) @unpack boundaries = cache_parabolic - @unpack neighbor_sides = boundaries + @unpack neighbor_sides, neighbor_ids = boundaries + boundaries_u = boundaries.u @threaded for boundary in eachboundary(dg, cache_parabolic) - element = boundaries.neighbor_ids[boundary] + element = neighbor_ids[boundary] if neighbor_sides[boundary] == 1 # element in -x direction of boundary for v in eachvariable(equations_parabolic) - # OBS! `boundaries.u` stores the interpolated *fluxes* and *not the solution*! - boundaries.u[1, v, boundary] = flux_viscous[v, nnodes(dg), element] + # OBS! `boundaries_u` stores the interpolated *fluxes* and *not the solution*! + boundaries_u[1, v, boundary] = flux_viscous[v, nnodes(dg), element] end else # Element in +x direction of boundary for v in eachvariable(equations_parabolic) - # OBS! `boundaries.u` stores the interpolated *fluxes* and *not the solution*! - boundaries.u[2, v, boundary] = flux_viscous[v, 1, element] + # OBS! `boundaries_u` stores the interpolated *fluxes* and *not the solution*! + boundaries_u[2, v, boundary] = flux_viscous[v, 1, element] end end end @@ -530,18 +535,15 @@ function create_cache_parabolic(mesh::TreeMesh{1}, elements = init_elements(leaf_cell_ids, mesh, equations_hyperbolic, dg.basis, RealT, uEltype) - n_vars = nvariables(equations_hyperbolic) - n_nodes = nnodes(elements) - n_elements = nelements(elements) - u_transformed = Array{uEltype}(undef, n_vars, n_nodes, n_elements) - gradients = similar(u_transformed) - flux_viscous = similar(u_transformed) + viscous_container = init_viscous_container(nvariables(equations_hyperbolic), + nnodes(elements), nelements(elements), + uEltype) interfaces = init_interfaces(leaf_cell_ids, mesh, elements) boundaries = init_boundaries(leaf_cell_ids, mesh, elements) - cache = (; elements, interfaces, boundaries, gradients, flux_viscous, u_transformed) + cache = (; elements, interfaces, boundaries, viscous_container) return cache end @@ -552,8 +554,10 @@ end # where f(u) is the inviscid flux and g(u) is the viscous flux. function apply_jacobian_parabolic!(du, mesh::TreeMesh{1}, equations::AbstractEquationsParabolic, dg::DG, cache) + @unpack inverse_jacobian = cache.elements + @threaded for element in eachelement(dg, cache) - factor = cache.elements.inverse_jacobian[element] + factor = inverse_jacobian[element] for i in eachnode(dg) for v in eachvariable(equations) diff --git a/src/solvers/dgsem_tree/dg_2d_parabolic.jl b/src/solvers/dgsem_tree/dg_2d_parabolic.jl index 0da25230380..3dbc55412ad 100644 --- a/src/solvers/dgsem_tree/dg_2d_parabolic.jl +++ b/src/solvers/dgsem_tree/dg_2d_parabolic.jl @@ -118,12 +118,13 @@ end function transform_variables!(u_transformed, u, mesh::Union{TreeMesh{2}, P4estMesh{2}}, equations_parabolic::AbstractEquationsParabolic, dg::DG, parabolic_scheme, cache, cache_parabolic) + transformation = gradient_variable_transformation(equations_parabolic) + @threaded for element in eachelement(dg, cache) # Calculate volume terms in one element for j in eachnode(dg), i in eachnode(dg) u_node = get_node_vars(u, equations_parabolic, dg, i, j, element) - u_transformed_node = gradient_variable_transformation(equations_parabolic)(u_node, - equations_parabolic) + u_transformed_node = transformation(u_node, equations_parabolic) set_node_vars!(u_transformed, u_transformed_node, equations_parabolic, dg, i, j, element) end @@ -168,30 +169,31 @@ function prolong2interfaces!(cache_parabolic, flux_viscous, equations_parabolic::AbstractEquationsParabolic, surface_integral, dg::DG, cache) @unpack interfaces = cache_parabolic - @unpack orientations = interfaces + @unpack orientations, neighbor_ids = interfaces + interfaces_u = interfaces.u flux_viscous_x, flux_viscous_y = flux_viscous @threaded for interface in eachinterface(dg, cache) - left_element = interfaces.neighbor_ids[1, interface] - right_element = interfaces.neighbor_ids[2, interface] + left_element = neighbor_ids[1, interface] + right_element = neighbor_ids[2, interface] if orientations[interface] == 1 # interface in x-direction for j in eachnode(dg), v in eachvariable(equations_parabolic) - # OBS! `interfaces.u` stores the interpolated *fluxes* and *not the solution*! - interfaces.u[1, v, j, interface] = flux_viscous_x[v, nnodes(dg), j, + # OBS! `interfaces_u` stores the interpolated *fluxes* and *not the solution*! + interfaces_u[1, v, j, interface] = flux_viscous_x[v, nnodes(dg), j, left_element] - interfaces.u[2, v, j, interface] = flux_viscous_x[v, 1, j, + interfaces_u[2, v, j, interface] = flux_viscous_x[v, 1, j, right_element] end else # if orientations[interface] == 2 # interface in y-direction for i in eachnode(dg), v in eachvariable(equations_parabolic) - # OBS! `interfaces.u` stores the interpolated *fluxes* and *not the solution*! - interfaces.u[1, v, i, interface] = flux_viscous_y[v, i, nnodes(dg), + # OBS! `interfaces_u` stores the interpolated *fluxes* and *not the solution*! + interfaces_u[1, v, i, interface] = flux_viscous_y[v, i, nnodes(dg), left_element] - interfaces.u[2, v, i, interface] = flux_viscous_y[v, i, 1, + interfaces_u[2, v, i, interface] = flux_viscous_y[v, i, 1, right_element] end end @@ -244,25 +246,26 @@ function prolong2boundaries!(cache_parabolic, flux_viscous, equations_parabolic::AbstractEquationsParabolic, surface_integral, dg::DG, cache) @unpack boundaries = cache_parabolic - @unpack orientations, neighbor_sides = boundaries + @unpack orientations, neighbor_sides, neighbor_ids = boundaries + boundaries_u = boundaries.u flux_viscous_x, flux_viscous_y = flux_viscous @threaded for boundary in eachboundary(dg, cache_parabolic) - element = boundaries.neighbor_ids[boundary] + element = neighbor_ids[boundary] if orientations[boundary] == 1 # boundary in x-direction if neighbor_sides[boundary] == 1 # element in -x direction of boundary for l in eachnode(dg), v in eachvariable(equations_parabolic) - # OBS! `boundaries.u` stores the interpolated *fluxes* and *not the solution*! - boundaries.u[1, v, l, boundary] = flux_viscous_x[v, nnodes(dg), l, + # OBS! `boundaries_u` stores the interpolated *fluxes* and *not the solution*! + boundaries_u[1, v, l, boundary] = flux_viscous_x[v, nnodes(dg), l, element] end else # Element in +x direction of boundary for l in eachnode(dg), v in eachvariable(equations_parabolic) - # OBS! `boundaries.u` stores the interpolated *fluxes* and *not the solution*! - boundaries.u[2, v, l, boundary] = flux_viscous_x[v, 1, l, element] + # OBS! `boundaries_u` stores the interpolated *fluxes* and *not the solution*! + boundaries_u[2, v, l, boundary] = flux_viscous_x[v, 1, l, element] end end else # if orientations[boundary] == 2 @@ -270,15 +273,15 @@ function prolong2boundaries!(cache_parabolic, flux_viscous, if neighbor_sides[boundary] == 1 # element in -y direction of boundary for l in eachnode(dg), v in eachvariable(equations_parabolic) - # OBS! `boundaries.u` stores the interpolated *fluxes* and *not the solution*! - boundaries.u[1, v, l, boundary] = flux_viscous_y[v, l, nnodes(dg), + # OBS! `boundaries_u` stores the interpolated *fluxes* and *not the solution*! + boundaries_u[1, v, l, boundary] = flux_viscous_y[v, l, nnodes(dg), element] end else # element in +y direction of boundary for l in eachnode(dg), v in eachvariable(equations_parabolic) - # OBS! `boundaries.u` stores the interpolated *fluxes* and *not the solution*! - boundaries.u[2, v, l, boundary] = flux_viscous_y[v, l, 1, element] + # OBS! `boundaries_u` stores the interpolated *fluxes* and *not the solution*! + boundaries_u[2, v, l, boundary] = flux_viscous_y[v, l, 1, element] end end end @@ -608,7 +611,7 @@ function prolong2mortars!(cache, flux_viscous::Tuple{AbstractArray, AbstractArra end # NOTE: Use analogy to "calc_mortar_flux!" for hyperbolic eqs with no nonconservative terms. -# Reasoning: "calc_interface_flux!" for parabolic part is implemented as the version for +# Reasoning: "calc_interface_flux!" for parabolic part is implemented as the version for # hyperbolic terms with conserved terms only, i.e., no nonconservative terms. function calc_mortar_flux!(surface_flux_values, mesh::TreeMesh{2}, @@ -934,8 +937,10 @@ end # where f(u) is the inviscid flux and g(u) is the viscous flux. function apply_jacobian_parabolic!(du, mesh::Union{TreeMesh{2}, P4estMesh{2}}, equations::AbstractEquationsParabolic, dg::DG, cache) + @unpack inverse_jacobian = cache.elements + @threaded for element in eachelement(dg, cache) - factor = cache.elements.inverse_jacobian[element] + factor = inverse_jacobian[element] for j in eachnode(dg), i in eachnode(dg) for v in eachvariable(equations) diff --git a/src/solvers/dgsem_tree/dg_3d_parabolic.jl b/src/solvers/dgsem_tree/dg_3d_parabolic.jl index 2745d312b37..9817e0e5f0e 100644 --- a/src/solvers/dgsem_tree/dg_3d_parabolic.jl +++ b/src/solvers/dgsem_tree/dg_3d_parabolic.jl @@ -118,12 +118,13 @@ end function transform_variables!(u_transformed, u, mesh::Union{TreeMesh{3}, P4estMesh{3}}, equations_parabolic::AbstractEquationsParabolic, dg::DG, parabolic_scheme, cache, cache_parabolic) + transformation = gradient_variable_transformation(equations_parabolic) + @threaded for element in eachelement(dg, cache) # Calculate volume terms in one element for k in eachnode(dg), j in eachnode(dg), i in eachnode(dg) u_node = get_node_vars(u, equations_parabolic, dg, i, j, k, element) - u_transformed_node = gradient_variable_transformation(equations_parabolic)(u_node, - equations_parabolic) + u_transformed_node = transformation(u_node, equations_parabolic) set_node_vars!(u_transformed, u_transformed_node, equations_parabolic, dg, i, j, k, element) end @@ -175,43 +176,44 @@ function prolong2interfaces!(cache_parabolic, flux_viscous, equations_parabolic::AbstractEquationsParabolic, surface_integral, dg::DG, cache) @unpack interfaces = cache_parabolic - @unpack orientations = interfaces + @unpack orientations, neighbor_ids = interfaces + interfaces_u = interfaces.u flux_viscous_x, flux_viscous_y, flux_viscous_z = flux_viscous @threaded for interface in eachinterface(dg, cache) - left_element = interfaces.neighbor_ids[1, interface] - right_element = interfaces.neighbor_ids[2, interface] + left_element = neighbor_ids[1, interface] + right_element = neighbor_ids[2, interface] if orientations[interface] == 1 # interface in x-direction for k in eachnode(dg), j in eachnode(dg), v in eachvariable(equations_parabolic) - # OBS! `interfaces.u` stores the interpolated *fluxes* and *not the solution*! - interfaces.u[1, v, j, k, interface] = flux_viscous_x[v, nnodes(dg), j, + # OBS! `interfaces_u` stores the interpolated *fluxes* and *not the solution*! + interfaces_u[1, v, j, k, interface] = flux_viscous_x[v, nnodes(dg), j, k, left_element] - interfaces.u[2, v, j, k, interface] = flux_viscous_x[v, 1, j, k, + interfaces_u[2, v, j, k, interface] = flux_viscous_x[v, 1, j, k, right_element] end elseif orientations[interface] == 2 # interface in y-direction for k in eachnode(dg), i in eachnode(dg), v in eachvariable(equations_parabolic) - # OBS! `interfaces.u` stores the interpolated *fluxes* and *not the solution*! - interfaces.u[1, v, i, k, interface] = flux_viscous_y[v, i, nnodes(dg), + # OBS! `interfaces_u` stores the interpolated *fluxes* and *not the solution*! + interfaces_u[1, v, i, k, interface] = flux_viscous_y[v, i, nnodes(dg), k, left_element] - interfaces.u[2, v, i, k, interface] = flux_viscous_y[v, i, 1, k, + interfaces_u[2, v, i, k, interface] = flux_viscous_y[v, i, 1, k, right_element] end else # if orientations[interface] == 3 # interface in z-direction for j in eachnode(dg), i in eachnode(dg), v in eachvariable(equations_parabolic) - # OBS! `interfaces.u` stores the interpolated *fluxes* and *not the solution*! - interfaces.u[1, v, i, j, interface] = flux_viscous_z[v, i, j, + # OBS! `interfaces_u` stores the interpolated *fluxes* and *not the solution*! + interfaces_u[1, v, i, j, interface] = flux_viscous_z[v, i, j, nnodes(dg), left_element] - interfaces.u[2, v, i, j, interface] = flux_viscous_z[v, i, j, 1, + interfaces_u[2, v, i, j, interface] = flux_viscous_z[v, i, j, 1, right_element] end end @@ -265,11 +267,12 @@ function prolong2boundaries!(cache_parabolic, flux_viscous, equations_parabolic::AbstractEquationsParabolic, surface_integral, dg::DG, cache) @unpack boundaries = cache_parabolic - @unpack orientations, neighbor_sides = boundaries + @unpack orientations, neighbor_sides, neighbor_ids = boundaries + boundaries_u = boundaries.u flux_viscous_x, flux_viscous_y, flux_viscous_z = flux_viscous @threaded for boundary in eachboundary(dg, cache_parabolic) - element = boundaries.neighbor_ids[boundary] + element = neighbor_ids[boundary] if orientations[boundary] == 1 # boundary in x-direction @@ -277,15 +280,15 @@ function prolong2boundaries!(cache_parabolic, flux_viscous, # element in -x direction of boundary for k in eachnode(dg), j in eachnode(dg), v in eachvariable(equations_parabolic) - # OBS! `boundaries.u` stores the interpolated *fluxes* and *not the solution*! - boundaries.u[1, v, j, k, boundary] = flux_viscous_x[v, nnodes(dg), + # OBS! `boundaries_u` stores the interpolated *fluxes* and *not the solution*! + boundaries_u[1, v, j, k, boundary] = flux_viscous_x[v, nnodes(dg), j, k, element] end else # Element in +x direction of boundary for k in eachnode(dg), j in eachnode(dg), v in eachvariable(equations_parabolic) - # OBS! `boundaries.u` stores the interpolated *fluxes* and *not the solution*! - boundaries.u[2, v, j, k, boundary] = flux_viscous_x[v, 1, j, k, + # OBS! `boundaries_u` stores the interpolated *fluxes* and *not the solution*! + boundaries_u[2, v, j, k, boundary] = flux_viscous_x[v, 1, j, k, element] end end @@ -295,8 +298,8 @@ function prolong2boundaries!(cache_parabolic, flux_viscous, # element in -y direction of boundary for k in eachnode(dg), i in eachnode(dg), v in eachvariable(equations_parabolic) - # OBS! `boundaries.u` stores the interpolated *fluxes* and *not the solution*! - boundaries.u[1, v, i, k, boundary] = flux_viscous_y[v, i, + # OBS! `boundaries_u` stores the interpolated *fluxes* and *not the solution*! + boundaries_u[1, v, i, k, boundary] = flux_viscous_y[v, i, nnodes(dg), k, element] end @@ -304,8 +307,8 @@ function prolong2boundaries!(cache_parabolic, flux_viscous, # element in +y direction of boundary for k in eachnode(dg), i in eachnode(dg), v in eachvariable(equations_parabolic) - # OBS! `boundaries.u` stores the interpolated *fluxes* and *not the solution*! - boundaries.u[2, v, i, k, boundary] = flux_viscous_y[v, i, 1, k, + # OBS! `boundaries_u` stores the interpolated *fluxes* and *not the solution*! + boundaries_u[2, v, i, k, boundary] = flux_viscous_y[v, i, 1, k, element] end end @@ -315,8 +318,8 @@ function prolong2boundaries!(cache_parabolic, flux_viscous, # element in -z direction of boundary for j in eachnode(dg), i in eachnode(dg), v in eachvariable(equations_parabolic) - # OBS! `boundaries.u` stores the interpolated *fluxes* and *not the solution*! - boundaries.u[1, v, i, j, boundary] = flux_viscous_z[v, i, j, + # OBS! `boundaries_u` stores the interpolated *fluxes* and *not the solution*! + boundaries_u[1, v, i, j, boundary] = flux_viscous_z[v, i, j, nnodes(dg), element] end @@ -324,8 +327,8 @@ function prolong2boundaries!(cache_parabolic, flux_viscous, # element in +z direction of boundary for j in eachnode(dg), i in eachnode(dg), v in eachvariable(equations_parabolic) - # OBS! `boundaries.u` stores the interpolated *fluxes* and *not the solution*! - boundaries.u[2, v, i, j, boundary] = flux_viscous_z[v, i, j, 1, + # OBS! `boundaries_u` stores the interpolated *fluxes* and *not the solution*! + boundaries_u[2, v, i, j, boundary] = flux_viscous_z[v, i, j, 1, element] end end @@ -820,7 +823,7 @@ function prolong2mortars!(cache, end # NOTE: Use analogy to "calc_mortar_flux!" for hyperbolic eqs with no nonconservative terms. -# Reasoning: "calc_interface_flux!" for parabolic part is implemented as the version for +# Reasoning: "calc_interface_flux!" for parabolic part is implemented as the version for # hyperbolic terms with conserved terms only, i.e., no nonconservative terms. function calc_mortar_flux!(surface_flux_values, mesh::TreeMesh{3}, @@ -1124,8 +1127,10 @@ end # where f(u) is the inviscid flux and g(u) is the viscous flux. function apply_jacobian_parabolic!(du, mesh::Union{TreeMesh{3}, P4estMesh{3}}, equations::AbstractEquationsParabolic, dg::DG, cache) + @unpack inverse_jacobian = cache.elements + @threaded for element in eachelement(dg, cache) - factor = cache.elements.inverse_jacobian[element] + factor = inverse_jacobian[element] for k in eachnode(dg), j in eachnode(dg), i in eachnode(dg) for v in eachvariable(equations) diff --git a/test/test_p4est_2d.jl b/test/test_p4est_2d.jl index c4ce2619e15..31dfe1d35a5 100644 --- a/test/test_p4est_2d.jl +++ b/test/test_p4est_2d.jl @@ -24,7 +24,7 @@ isdir(outdir) && rm(outdir, recursive=true) l2 = [3.198940059144588e-5], linf = [0.00030636069494005547]) - # Ensure that we do not have excessive memory allocations + # Ensure that we do not have excessive memory allocations # (e.g., from type instabilities) let t = sol.t[end] @@ -102,6 +102,15 @@ isdir(outdir) && rm(outdir, recursive=true) l2 = [0.020291447969983396, 0.017479614254319948, 0.011387644425613437, 0.0514420126021293], linf = [0.3582779022370579, 0.32073537890751663, 0.221818049107692, 0.9209559420400415], tspan = (0.0, 0.15)) + + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end end @trixi_testset "elixir_euler_forward_step_amr.jl" begin diff --git a/test/test_parabolic_1d.jl b/test/test_parabolic_1d.jl index 06a55100d62..3c2b8855ce8 100644 --- a/test/test_parabolic_1d.jl +++ b/test/test_parabolic_1d.jl @@ -20,6 +20,28 @@ isdir(outdir) && rm(outdir, recursive=true) ) end + @trixi_testset "TreeMesh1D: elixir_advection_diffusion.jl (AMR)" begin + @test_trixi_include(joinpath(examples_dir(), "tree_1d_dgsem", "elixir_advection_diffusion.jl"), + tspan=(0.0, 0.0), initial_refinement_level = 5) + tspan=(0.0, 1.0) + ode = semidiscretize(semi, tspan) + amr_controller = ControllerThreeLevel(semi, IndicatorMax(semi, variable=first), + base_level=4, + med_level=5, med_threshold=0.1, + max_level=6, max_threshold=0.6) + amr_callback = AMRCallback(semi, amr_controller, + interval=5, + adapt_initial_condition=true) + + # Create a CallbackSet to collect all callbacks such that they can be passed to the ODE solver + callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, amr_callback) + sol = solve(ode, KenCarp4(autodiff=false), abstol=time_abs_tol, reltol=time_int_tol, + save_everystep=false, callback=callbacks) + l2_error, linf_error = analysis_callback(sol) + @test l2_error ≈ [6.4878111416468355e-6] + @test linf_error ≈ [3.258075790424364e-5] + end + @trixi_testset "TreeMesh1D: elixir_navierstokes_convergence_periodic.jl" begin @test_trixi_include(joinpath(examples_dir(), "tree_1d_dgsem", "elixir_navierstokes_convergence_periodic.jl"), l2 = [0.0001133835907077494, 6.226282245610444e-5, 0.0002820171699999139], @@ -53,6 +75,25 @@ isdir(outdir) && rm(outdir, recursive=true) linf = [0.002754803146635787, 0.0028567714697580906, 0.012941794048176192] ) end + + @trixi_testset "TreeMesh1D: elixir_navierstokes_convergence_walls_amr.jl" begin + @test_trixi_include(joinpath(examples_dir(), "tree_1d_dgsem", "elixir_navierstokes_convergence_walls_amr.jl"), + equations_parabolic = CompressibleNavierStokesDiffusion1D(equations, mu=mu(), + Prandtl=prandtl_number()), + l2 = [2.527877257772131e-5, 2.5539911566937718e-5, 0.0001211860451244785], + linf = [0.00014663867588948776, 0.00019422448348348196, 0.0009556439394007299] + ) + end + + @trixi_testset "TreeMesh1D: elixir_navierstokes_convergence_walls_amr.jl: GradientVariablesEntropy" begin + @test_trixi_include(joinpath(examples_dir(), "tree_1d_dgsem", "elixir_navierstokes_convergence_walls_amr.jl"), + equations_parabolic = CompressibleNavierStokesDiffusion1D(equations, mu=mu(), + Prandtl=prandtl_number(), + gradient_variables = GradientVariablesEntropy()), + l2 = [2.4593699163175966e-5, 2.392863645712634e-5, 0.00011252526651714956], + linf = [0.00011850555445525046, 0.0001898777490968537, 0.0009597561467877824] + ) + end end # Clean up afterwards: delete Trixi output directory diff --git a/test/test_parabolic_2d.jl b/test/test_parabolic_2d.jl index 1564a33dc41..3fff4382cd1 100644 --- a/test/test_parabolic_2d.jl +++ b/test/test_parabolic_2d.jl @@ -143,9 +143,9 @@ isdir(outdir) && rm(outdir, recursive=true) callbacks = CallbackSet(summary_callback, alive_callback, analysis_callback) sol = solve(ode, RDPK3SpFSAL49(); abstol=time_int_tol, reltol=time_int_tol, ode_default_options()..., callback=callbacks) - ac_sol = analysis_callback(sol) - @test ac_sol.l2[1] ≈ 1.67452550744728e-6 - @test ac_sol.linf[1] ≈ 7.905059166368744e-6 + l2_error, linf_error = analysis_callback(sol) + @test l2_error ≈ [1.67452550744728e-6] + @test linf_error ≈ [7.905059166368744e-6] # Ensure that we do not have excessive memory allocations # (e.g., from type instabilities) @@ -229,9 +229,9 @@ isdir(outdir) && rm(outdir, recursive=true) callbacks = CallbackSet(summary_callback, alive_callback, analysis_callback) sol = solve(ode, RDPK3SpFSAL49(); abstol=time_int_tol, reltol=time_int_tol, dt = 1e-5, ode_default_options()..., callback=callbacks) - ac_sol = analysis_callback(sol) - @test ac_sol.l2 ≈ [0.00024296959173852447; 0.0002093263158670915; 0.0005390572390977262; 0.00026753561392341537] - @test ac_sol.linf ≈ [0.0016210102053424436; 0.002593287648655501; 0.002953907343823712; 0.002077119120180271] + l2_error, linf_error = analysis_callback(sol) + @test l2_error ≈ [0.00024296959173852447; 0.0002093263158670915; 0.0005390572390977262; 0.00026753561392341537] + @test linf_error ≈ [0.0016210102053424436; 0.002593287648655501; 0.002953907343823712; 0.002077119120180271] end @trixi_testset "TreeMesh2D: elixir_navierstokes_lid_driven_cavity.jl" begin diff --git a/test/test_parabolic_3d.jl b/test/test_parabolic_3d.jl index d607962afa0..86076460294 100644 --- a/test/test_parabolic_3d.jl +++ b/test/test_parabolic_3d.jl @@ -85,7 +85,7 @@ isdir(outdir) && rm(outdir, recursive=true) num_leafs = length(LLID) @assert num_leafs % 16 == 0 Trixi.refine!(mesh.tree, LLID[1:Int(num_leafs/16)]) - tspan=(0.0, 1.0) + tspan=(0.0, 0.25) semi = SemidiscretizationHyperbolicParabolic(mesh, (equations, equations_parabolic), initial_condition, solver; boundary_conditions=(boundary_conditions, boundary_conditions_parabolic), source_terms=source_terms_navier_stokes_convergence_test) @@ -94,9 +94,9 @@ isdir(outdir) && rm(outdir, recursive=true) callbacks = CallbackSet(summary_callback, alive_callback, analysis_callback) sol = solve(ode, RDPK3SpFSAL49(); abstol=time_int_tol, reltol=time_int_tol, dt = 1e-5, ode_default_options()..., callback=callbacks) - ac_sol = analysis_callback(sol) - @test ac_sol.l2 ≈ [0.0003991794175622818; 0.0008853745163670504; 0.0010658655552066817; 0.0008785559918324284; 0.001403163458422815] - @test ac_sol.linf ≈ [0.0035306410538458177; 0.01505692306169911; 0.008862444161110705; 0.015065647972869856; 0.030402714743065218] + l2_error, linf_error = analysis_callback(sol) + @test l2_error ≈ [0.0003109336253407314, 0.0006473493036803503, 0.0007705277238213672, 0.0006280517917198335, 0.000903927789884075] + @test linf_error ≈ [0.0023694155365339142, 0.010634932622402863, 0.006772070862236412, 0.010640551561726901, 0.019256819038719897] end @trixi_testset "TreeMesh3D: elixir_navierstokes_taylor_green_vortex.jl" begin @@ -114,7 +114,7 @@ isdir(outdir) && rm(outdir, recursive=true) num_leafs = length(LLID) @assert num_leafs % 32 == 0 Trixi.refine!(mesh.tree, LLID[1:Int(num_leafs/32)]) - tspan=(0.0, 10.0) + tspan=(0.0, 0.1) semi = SemidiscretizationHyperbolicParabolic(mesh, (equations, equations_parabolic), initial_condition, solver) ode = semidiscretize(semi, tspan) @@ -127,9 +127,9 @@ isdir(outdir) && rm(outdir, recursive=true) sol = solve(ode, CarpenterKennedy2N54(williamson_condition=false), dt=5e-3, save_everystep=false, callback=callbacks); - ac_sol = analysis_callback(sol) - @test ac_sol.l2 ≈ [0.0013666103707729502; 0.2313581629543744; 0.2308164306264533; 0.17460246787819503; 0.28121914446544005] - @test ac_sol.linf ≈ [0.006938093883741336; 1.028235074139312; 1.0345438209717241; 1.0821111605203542; 1.2669636522564645] + l2_error, linf_error = analysis_callback(sol) + @test l2_error ≈ [7.314319856736271e-5, 0.006266480163542894, 0.006266489911815533, 0.008829222305770226, 0.0032859166842329228] + @test linf_error ≈ [0.0002943968186086554, 0.013876261980614757, 0.013883619864959451, 0.025201279960491936, 0.018679364985388247] # Ensure that we do not have excessive memory allocations # (e.g., from type instabilities)