From 95a021d7534053cdf44ace56efd67d3fc65828df Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Mon, 8 Apr 2024 13:04:10 +0200 Subject: [PATCH 01/81] Wrap from_abaqus routines. --- src/meshes/p4est_mesh.jl | 44 ++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/src/meshes/p4est_mesh.jl b/src/meshes/p4est_mesh.jl index abe9d9345b5..61713867e0f 100644 --- a/src/meshes/p4est_mesh.jl +++ b/src/meshes/p4est_mesh.jl @@ -387,11 +387,43 @@ function P4estMesh{NDIMS}(meshfile::String; p4est_partition_allow_for_coarsening) end +# Wrapper for `p4est_connectivity_from_hohqmesh_abaqus`. The latter is used +# by `T8codeMesh`, too. +function p4est_mesh_from_hohqmesh_abaqus(meshfile, initial_refinement_level, + n_dimensions, RealT) + connectivity, tree_node_coordinates, nodes, boundary_names = p4est_connectivity_from_hohqmesh_abaqus(meshfile, + initial_refinement_level, + n_dimensions, + RealT) + + p4est = new_p4est(connectivity, initial_refinement_level) + + return p4est, tree_node_coordinates, nodes, boundary_names +end + +# Wrapper for `p4est_connectivity_from_standard_abaqus`. The latter is used +# by `T8codeMesh`, too. +function p4est_mesh_from_standard_abaqus(meshfile, mapping, polydeg, + initial_refinement_level, n_dimensions, RealT, + boundary_symbols) + connectivity, tree_node_coordinates, nodes, boundary_names = p4est_connectivity_from_standard_abaqus(meshfile, + mapping, + polydeg, + initial_refinement_level, + n_dimensions, + RealT, + boundary_symbols) + + p4est = new_p4est(connectivity, initial_refinement_level) + + return p4est, tree_node_coordinates, nodes, boundary_names +end + # Create the mesh connectivity, mapped node coordinates within each tree, reference nodes in [-1,1] # and a list of boundary names for the `P4estMesh`. High-order boundary curve information as well as # the boundary names on each tree are provided by the `meshfile` created by # [`HOHQMesh.jl`](https://github.com/trixi-framework/HOHQMesh.jl). -function p4est_mesh_from_hohqmesh_abaqus(meshfile, initial_refinement_level, +function p4est_connectivity_from_hohqmesh_abaqus(meshfile, initial_refinement_level, n_dimensions, RealT) # Create the mesh connectivity using `p4est` connectivity = read_inp_p4est(meshfile, Val(n_dimensions)) @@ -440,16 +472,14 @@ function p4est_mesh_from_hohqmesh_abaqus(meshfile, initial_refinement_level, file_idx += 1 end - p4est = new_p4est(connectivity, initial_refinement_level) - - return p4est, tree_node_coordinates, nodes, boundary_names + return connectivity, tree_node_coordinates, nodes, boundary_names end # Create the mesh connectivity, mapped node coordinates within each tree, reference nodes in [-1,1] # and a list of boundary names for the `P4estMesh`. The tree node coordinates are computed according to # the `mapping` passed to this function using polynomial interpolants of degree `polydeg`. All boundary # names are given the name `:all`. -function p4est_mesh_from_standard_abaqus(meshfile, mapping, polydeg, +function p4est_connectivity_from_standard_abaqus(meshfile, mapping, polydeg, initial_refinement_level, n_dimensions, RealT, boundary_symbols) # Create the mesh connectivity using `p4est` @@ -474,8 +504,6 @@ function p4est_mesh_from_standard_abaqus(meshfile, mapping, polydeg, calc_tree_node_coordinates!(tree_node_coordinates, nodes, mapping, vertices, tree_to_vertex) - p4est = new_p4est(connectivity, initial_refinement_level) - if boundary_symbols === nothing # There's no simple and generic way to distinguish boundaries without any information given. # Name all of them :all. @@ -495,7 +523,7 @@ function p4est_mesh_from_standard_abaqus(meshfile, mapping, polydeg, Val(n_dimensions)) end - return p4est, tree_node_coordinates, nodes, boundary_names + return connectivity, tree_node_coordinates, nodes, boundary_names end function parse_elements(meshfile, n_trees, n_dims) From 178ec183cc607e6a5c0ce074cfc6cf1483d12de8 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Mon, 8 Apr 2024 13:05:17 +0200 Subject: [PATCH 02/81] Implement geometry data transfer from t8code to Trixi. --- src/Trixi.jl | 2 + src/auxiliary/auxiliary.jl | 11 + src/meshes/t8code_mesh.jl | 451 ++++++++++++++++++++++++------------- test/test_t8code_3d.jl | 2 +- 4 files changed, 304 insertions(+), 162 deletions(-) diff --git a/src/Trixi.jl b/src/Trixi.jl index 883f8d66f07..8c4a987ebd5 100644 --- a/src/Trixi.jl +++ b/src/Trixi.jl @@ -288,6 +288,8 @@ export PlotData1D, PlotData2D, ScalarPlotData2D, getmesh, adapt_to_mesh_level!, adapt_to_mesh_level, iplot, iplot! +export GmshFile, AbaqusFile + function __init__() init_mpi() diff --git a/src/auxiliary/auxiliary.jl b/src/auxiliary/auxiliary.jl index 972a748c56b..34e7ccbbfdb 100644 --- a/src/auxiliary/auxiliary.jl +++ b/src/auxiliary/auxiliary.jl @@ -370,4 +370,15 @@ function download(src_url, file_path) return file_path end + +abstract type MeshFile{NDIMS} end + +struct GmshFile{NDIMS} <: MeshFile{NDIMS} + path :: String +end + +struct AbaqusFile{NDIMS} <: MeshFile{NDIMS} + path :: String +end + end # @muladd diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index cb2ac787e14..72f7cf337b1 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -26,7 +26,7 @@ mutable struct T8codeMesh{NDIMS, RealT <: Real, IsParallel, NDIMSP2, NNODES} <: nmpiinterfaces :: Int nmpimortars :: Int - function T8codeMesh{NDIMS}(forest, tree_node_coordinates, nodes, + function T8codeMesh{NDIMS}(forest :: Ptr{t8_forest}, tree_node_coordinates, nodes, boundary_names, current_filename) where {NDIMS} is_parallel = mpi_isparallel() ? True() : False() @@ -100,6 +100,122 @@ function Base.show(io::IO, ::MIME"text/plain", mesh::T8codeMesh) end end +""" + T8codeMesh(forest, boundary_names; polydeg, mapping=identity) + +Create a 'T8codeMesh'. + +# Arguments +- 'forest': Pointer to a commited forest. +- 'boundary_names': List of boundary names. +- 'polydeg::Integer': Polynomial degree used to store the geometry of the mesh. + The mapping will be approximated by an interpolation polynomial + of the specified degree for each tree. +- `mapping`: a function of `NDIMS` variables to describe the mapping that transforms + the imported mesh to the physical domain. Use `nothing` for the identity map. +""" +function T8codeMesh{NDIMS, RealT}(forest :: Ptr{t8_forest}, boundary_names; polydeg = 1, mapping = nothing) where {NDIMS, RealT} + # In t8code reference space is [0,1]. + basis = LobattoLegendreBasis(RealT, polydeg) + nodes = 0.5 .* (basis.nodes .+ 1.0) + + cmesh = t8_forest_get_cmesh(forest) + num_trees = t8_forest_get_num_global_trees(forest) + + tree_node_coordinates = Array{RealT, NDIMS + 2}(undef, NDIMS, + ntuple(_ -> length(nodes), NDIMS)..., + num_trees) + + coords_ref = Vector{Cdouble}(undef, 3) + + # Calculate node coordinates of reference mesh. + if NDIMS == 2 + num_corners = 4 # quadrilateral + + # Testing for negative element volumes. + verts = zeros(3, num_corners) + for itree in 1:num_trees + veptr = t8_cmesh_get_tree_vertices(cmesh, itree-1) + + # Note, `verts = unsafe_wrap(Array, veptr, (3, 1 << NDIMS))` + # sometimes does not work since `veptr` is not necessarily properly + # aligned to 8 bytes. + for icorner in 1:num_corners + verts[1, icorner] = unsafe_load(veptr, (icorner - 1) * 3 + 1) + verts[2, icorner] = unsafe_load(veptr, (icorner - 1) * 3 + 2) + end + + # Check if tree's node ordering is right-handed or print a warning. + let z = zero(eltype(verts)), o = one(eltype(verts)) + u = verts[:, 2] - verts[:, 1] + v = verts[:, 3] - verts[:, 1] + w = [z, z, o] + + # Triple product gives signed volume of spanned parallelepiped. + vol = dot(cross(u, v), w) + + if vol < z + @warn "Discovered negative volumes in `cmesh`: vol = $vol" + end + end + + # Query geometry data from t8code. + for j in eachindex(nodes), i in eachindex(nodes) + coords_ref[1] = nodes[i] + coords_ref[2] = nodes[j] + coords_ref[3] = 0.0 + t8_geometry_evaluate(cmesh, itree-1, coords_ref, 1, @view(tree_node_coordinates[:, i, j, itree])) + end + end + + elseif NDIMS == 3 + num_corners = 8 # hexahedron + + # Testing for negative element volumes. + verts = zeros(3, num_corners) + for itree in 1:num_trees + veptr = t8_cmesh_get_tree_vertices(cmesh, itree-1) + + # Note, `verts = unsafe_wrap(Array, veptr, (3, 1 << NDIMS))` + # sometimes does not work since `veptr` is not necessarily properly + # aligned to 8 bytes. + for icorner in 1:num_corners + verts[1, icorner] = unsafe_load(veptr, (icorner - 1) * 3 + 1) + verts[2, icorner] = unsafe_load(veptr, (icorner - 1) * 3 + 2) + verts[3, icorner] = unsafe_load(veptr, (icorner - 1) * 3 + 3) + end + + # Check if tree's node ordering is right-handed or print a warning. + let z = zero(eltype(verts)) + u = verts[:, 2] - verts[:, 1] + v = verts[:, 3] - verts[:, 1] + w = verts[:, 5] - verts[:, 1] + + # Triple product gives signed volume of spanned parallelepiped. + vol = dot(cross(u, v), w) + + if vol < z + @warn "Discovered negative volumes in `cmesh`: vol = $vol" + end + end + + # Query geometry data from t8code. + for k in eachindex(nodes), j in eachindex(nodes), i in eachindex(nodes) + coords_ref[1] = nodes[i] + coords_ref[2] = nodes[j] + coords_ref[3] = nodes[k] + t8_geometry_evaluate(cmesh, itree-1, coords_ref, 1, @view(tree_node_coordinates[:, i, j, k, itree])) + end + end + end + + # Apply user defined mapping. + map_node_coordinates!(tree_node_coordinates, mapping) + + return T8codeMesh{NDIMS}(forest, tree_node_coordinates, basis.nodes, + boundary_names, "") +end + """ T8codeMesh(trees_per_dimension; polydeg, mapping=identity, RealT=Float64, initial_refinement_level=0, periodicity=true) @@ -187,57 +303,10 @@ function T8codeMesh(trees_per_dimension; polydeg = 1, forest = t8_forest_new_uniform(cmesh, scheme, initial_refinement_level, do_face_ghost, mpi_comm()) - basis = LobattoLegendreBasis(RealT, polydeg) - nodes = basis.nodes - - num_trees = t8_cmesh_get_num_trees(cmesh) - - tree_node_coordinates = Array{RealT, NDIMS + 2}(undef, NDIMS, - ntuple(_ -> length(nodes), NDIMS)..., - num_trees) - - # Get cell length in reference mesh: Omega_ref = [-1,1]^NDIMS. - dx = [2 / n for n in trees_per_dimension] - # Non-periodic boundaries. boundary_names = fill(Symbol("---"), 2 * NDIMS, prod(trees_per_dimension)) - if mapping === nothing - mapping_ = coordinates2mapping(ntuple(_ -> -1.0, NDIMS), ntuple(_ -> 1.0, NDIMS)) - else - mapping_ = mapping - end - - for itree in 1:num_trees - veptr = t8_cmesh_get_tree_vertices(cmesh, itree - 1) - verts = unsafe_wrap(Array, veptr, (3, 1 << NDIMS)) - - # Calculate node coordinates of reference mesh. - if NDIMS == 2 - cell_x_offset = (verts[1, 1] - 0.5 * (trees_per_dimension[1] - 1)) * dx[1] - cell_y_offset = (verts[2, 1] - 0.5 * (trees_per_dimension[2] - 1)) * dx[2] - - for j in eachindex(nodes), i in eachindex(nodes) - tree_node_coordinates[:, i, j, itree] .= mapping_(cell_x_offset + - dx[1] * nodes[i] / 2, - cell_y_offset + - dx[2] * nodes[j] / 2) - end - elseif NDIMS == 3 - cell_x_offset = (verts[1, 1] - 0.5 * (trees_per_dimension[1] - 1)) * dx[1] - cell_y_offset = (verts[2, 1] - 0.5 * (trees_per_dimension[2] - 1)) * dx[2] - cell_z_offset = (verts[3, 1] - 0.5 * (trees_per_dimension[3] - 1)) * dx[3] - - for k in eachindex(nodes), j in eachindex(nodes), i in eachindex(nodes) - tree_node_coordinates[:, i, j, k, itree] .= mapping_(cell_x_offset + - dx[1] * nodes[i] / 2, - cell_y_offset + - dx[2] * nodes[j] / 2, - cell_z_offset + - dx[3] * nodes[k] / 2) - end - end - + for itree in 1:t8_forest_get_num_global_trees(forest) if !periodicity[1] boundary_names[1, itree] = :x_neg boundary_names[2, itree] = :x_pos @@ -256,8 +325,11 @@ function T8codeMesh(trees_per_dimension; polydeg = 1, end end - return T8codeMesh{NDIMS}(forest, tree_node_coordinates, nodes, - boundary_names, "") + # Note, `p*est_connectivity_new_brick` convers a domain of `[0,nx] x [0,ny] x ....`. + # Hence, transform mesh coordinates to reference space [-1,1]^NDIMS before applying user defined mapping. + mapping_(xyz...) = mapping((x * 2.0/tpd - 1.0 for (x,tpd) in zip(xyz, trees_per_dimension))...) + + return T8codeMesh{NDIMS, RealT}(forest, boundary_names; polydeg = polydeg, mapping = mapping_) end """ @@ -295,106 +367,10 @@ function T8codeMesh(cmesh::Ptr{t8_cmesh}; forest = t8_forest_new_uniform(cmesh, scheme, initial_refinement_level, do_face_ghost, mpi_comm()) - basis = LobattoLegendreBasis(RealT, polydeg) - nodes = basis.nodes - - num_trees = t8_cmesh_get_num_trees(cmesh) - - tree_node_coordinates = Array{RealT, NDIMS + 2}(undef, NDIMS, - ntuple(_ -> length(nodes), NDIMS)..., - num_trees) - - nodes_in = [-1.0, 1.0] - matrix = polynomial_interpolation_matrix(nodes_in, nodes) - - num_local_trees = t8_cmesh_get_num_local_trees(cmesh) - - if NDIMS == 2 - data_in = Array{RealT, 3}(undef, 2, 2, 2) - tmp1 = zeros(RealT, 2, length(nodes), length(nodes_in)) - verts = zeros(3, 4) - - for itree in 0:(num_local_trees - 1) - veptr = t8_cmesh_get_tree_vertices(cmesh, itree) - - # Note, `verts = unsafe_wrap(Array, veptr, (3, 1 << NDIMS))` - # sometimes does not work since `veptr` is not necessarily properly - # aligned to 8 bytes. - for icorner in 1:4 - verts[1, icorner] = unsafe_load(veptr, (icorner - 1) * 3 + 1) - verts[2, icorner] = unsafe_load(veptr, (icorner - 1) * 3 + 2) - end - - # Check if tree's node ordering is right-handed or print a warning. - let z = zero(eltype(verts)), o = one(eltype(verts)) - u = verts[:, 2] - verts[:, 1] - v = verts[:, 3] - verts[:, 1] - w = [z, z, o] - - # Triple product gives signed volume of spanned parallelepiped. - vol = dot(cross(u, v), w) - - if vol < z - @warn "Discovered negative volumes in `cmesh`: vol = $vol" - end - end + # There's no simple and generic way to distinguish boundaries, yet. Name all of them :all. + boundary_names = fill(:all, 2 * NDIMS, t8_cmesh_get_num_trees(cmesh)) - # Tree vertices are stored in z-order. - @views data_in[:, 1, 1] .= verts[1:2, 1] - @views data_in[:, 2, 1] .= verts[1:2, 2] - @views data_in[:, 1, 2] .= verts[1:2, 3] - @views data_in[:, 2, 2] .= verts[1:2, 4] - - # Interpolate corner coordinates to specified nodes. - multiply_dimensionwise!(view(tree_node_coordinates, :, :, :, itree + 1), - matrix, matrix, - data_in, - tmp1) - end - - elseif NDIMS == 3 - data_in = Array{RealT, 4}(undef, 3, 2, 2, 2) - tmp1 = zeros(RealT, 3, length(nodes), length(nodes_in), length(nodes_in)) - verts = zeros(3, 8) - - for itree in 0:(num_trees - 1) - veptr = t8_cmesh_get_tree_vertices(cmesh, itree) - - # Note, `verts = unsafe_wrap(Array, veptr, (3, 1 << NDIMS))` - # sometimes does not work since `veptr` is not necessarily properly - # aligned to 8 bytes. - for icorner in 1:8 - verts[1, icorner] = unsafe_load(veptr, (icorner - 1) * 3 + 1) - verts[2, icorner] = unsafe_load(veptr, (icorner - 1) * 3 + 2) - verts[3, icorner] = unsafe_load(veptr, (icorner - 1) * 3 + 3) - end - - # Tree vertices are stored in z-order. - @views data_in[:, 1, 1, 1] .= verts[1:3, 1] - @views data_in[:, 2, 1, 1] .= verts[1:3, 2] - @views data_in[:, 1, 2, 1] .= verts[1:3, 3] - @views data_in[:, 2, 2, 1] .= verts[1:3, 4] - - @views data_in[:, 1, 1, 2] .= verts[1:3, 5] - @views data_in[:, 2, 1, 2] .= verts[1:3, 6] - @views data_in[:, 1, 2, 2] .= verts[1:3, 7] - @views data_in[:, 2, 2, 2] .= verts[1:3, 8] - - # Interpolate corner coordinates to specified nodes. - multiply_dimensionwise!(view(tree_node_coordinates, :, :, :, :, itree + 1), - matrix, matrix, matrix, - data_in, - tmp1) - end - end - - map_node_coordinates!(tree_node_coordinates, mapping) - - # There's no simple and generic way to distinguish boundaries. Name all of them :all. - boundary_names = fill(:all, 2 * NDIMS, num_trees) - - return T8codeMesh{NDIMS}(forest, tree_node_coordinates, nodes, - boundary_names, "") + return T8codeMesh{NDIMS, RealT}(forest, boundary_names; polydeg = polydeg, mapping = mapping) end """ @@ -446,35 +422,188 @@ function T8codeMesh(conn::Ptr{p8est_connectivity}; kwargs...) end """ - T8codeMesh(meshfile::String, ndims; kwargs...) + T8codeMesh(meshfile::String, NDIMS; kwargs...) + +Main mesh constructor for the `T8codeMesh` that imports an unstructured, conforming +mesh from either a Gmsh mesh file (`.msh`) or Abaqus mesh file (`.inp`) which is determined +by the file extension. + +# Arguments +- `filepath::String`: path to a Gmsh or Abaqus mesh file. +- `NDIMS`: Mesh file dimension: `2` or `3`. + +# Optional Keyword Arguments +- `mapping`: A function of `NDIMS` variables to describe the mapping that transforms + the imported mesh to the physical domain. Use `nothing` for the identity map. +- `polydeg::Integer`: Polynomial degree used to store the geometry of the mesh. + The mapping will be approximated by an interpolation polynomial + of the specified degree for each tree. + The default of `1` creates an uncurved geometry. Use a higher value if the mapping + will curve the imported uncurved mesh. +- `RealT::Type`: The type that should be used for coordinates. +- `initial_refinement_level::Integer`: Refine the mesh uniformly to this level before the simulation starts. +""" +function T8codeMesh(filepath::String, NDIMS; kwargs...) + # Prevent `t8code` from crashing Julia if the file doesn't exist. + @assert isfile(filepath) + + meshfile_prefix, meshfile_suffix = splitext(filepath) + + file_extension = lowercase(meshfile_suffix) + + if file_extension == ".msh" + return T8codeMesh(GmshFile{NDIMS}(filepath); kwargs...) + end + + if file_extension == ".inp" + return T8codeMesh(AbaqusFile{NDIMS}(filepath); kwargs...) + end + + throw("Unknown file extension: " * file_extension) +end + +""" + T8codeMesh(meshfile::GmshFile{NDIMS}; kwargs...) Main mesh constructor for the `T8codeMesh` that imports an unstructured, conforming mesh from a Gmsh mesh file (`.msh`). # Arguments -- `meshfile::String`: path to a Gmsh mesh file. -- `ndims`: Mesh file dimension: `2` or `3`. -- `mapping`: a function of `NDIMS` variables to describe the mapping that transforms +- `meshfile::GmshFile{NDIMS}`: Gmsh mesh file object of dimension NDIMS and give `path` to the file. + +# Optional Keyword Arguments +- `mapping`: A function of `NDIMS` variables to describe the mapping that transforms the imported mesh to the physical domain. Use `nothing` for the identity map. -- `polydeg::Integer`: polynomial degree used to store the geometry of the mesh. +- `polydeg::Integer`: Polynomial degree used to store the geometry of the mesh. The mapping will be approximated by an interpolation polynomial of the specified degree for each tree. The default of `1` creates an uncurved geometry. Use a higher value if the mapping will curve the imported uncurved mesh. -- `RealT::Type`: the type that should be used for coordinates. -- `initial_refinement_level::Integer`: refine the mesh uniformly to this level before the simulation starts. +- `RealT::Type`: The type that should be used for coordinates. +- `initial_refinement_level::Integer`: Refine the mesh uniformly to this level before the simulation starts. """ -function T8codeMesh(meshfile::String, ndims; kwargs...) +function T8codeMesh(meshfile::GmshFile{NDIMS}; kwargs...) where NDIMS # Prevent `t8code` from crashing Julia if the file doesn't exist. - @assert isfile(meshfile) + @assert isfile(meshfile.path) - meshfile_prefix, meshfile_suffix = splitext(meshfile) + meshfile_prefix, meshfile_suffix = splitext(meshfile.path) - cmesh = t8_cmesh_from_msh_file(meshfile_prefix, 0, mpi_comm(), ndims, 0, 0) + cmesh = t8_cmesh_from_msh_file(meshfile_prefix, 0, mpi_comm(), NDIMS, 0, 0) return T8codeMesh(cmesh; kwargs...) end +""" + T8codeMesh(meshfile::AbaqusFile{NDIMS}; + mapping=nothing, polydeg=1, RealT=Float64, + initial_refinement_level=0, unsaved_changes=true, + boundary_symbols = nothing) + +Main mesh constructor for the `T8codeMesh` that imports an unstructured, conforming +mesh from an Abaqus mesh file (`.inp`). + +To create a curved unstructured mesh `T8codeMesh` two strategies are available: + +- `HOHQMesh Abaqus`: High-order, curved boundary information created by + [`HOHQMesh.jl`](https://github.com/trixi-framework/HOHQMesh.jl) is + available in the `meshfile`. The mesh polynomial degree `polydeg` + of the boundaries is provided from the `meshfile`. The computation of + the mapped tree coordinates is done with transfinite interpolation + with linear blending similar to [`UnstructuredMesh2D`](@ref). Boundary name + information is also parsed from the `meshfile` such that different boundary + conditions can be set at each named boundary on a given tree. + +- `Standard Abaqus`: By default, with `mapping=nothing` and `polydeg=1`, this creates a + straight-sided from the information parsed from the `meshfile`. If a mapping + function is specified then it computes the mapped tree coordinates via polynomial + interpolants with degree `polydeg`. The mesh created by this function will only + have one boundary `:all` if `boundary_symbols` is not specified. + If `boundary_symbols` is specified the mesh file will be parsed for nodesets defining + the boundary nodes from which boundary edges (2D) and faces (3D) will be assigned. + +Note that the `mapping` and `polydeg` keyword arguments are only used by the `HOHQMesh Abaqus` option. +The `Standard Abaqus` routine obtains the mesh `polydeg` directly from the `meshfile` +and constructs the transfinite mapping internally. + +The particular strategy is selected according to the header present in the `meshfile` where +the constructor checks whether or not the `meshfile` was created with +[HOHQMesh.jl](https://github.com/trixi-framework/HOHQMesh.jl). +If the Abaqus file header is not present in the `meshfile` then the `T8codeMesh` is created +by `Standard Abaqus`. + +The default keyword argument `initial_refinement_level=0` corresponds to a forest +where the number of trees is the same as the number of elements in the original `meshfile`. +Increasing the `initial_refinement_level` allows one to uniformly refine the base mesh given +in the `meshfile` to create a forest with more trees before the simulation begins. +For example, if a two-dimensional base mesh contains 25 elements then setting +`initial_refinement_level=1` creates an initial forest of `2^2 * 25 = 100` trees. + +# Arguments +- `meshfile::AbaqusFile{NDIMS}`: Abaqus mesh file object of dimension NDIMS and give `path` to the file. + +# Optional Keyword Arguments +- `mapping`: A function of `NDIMS` variables to describe the mapping that transforms + the imported mesh to the physical domain. Use `nothing` for the identity map. +- `polydeg::Integer`: Polynomial degree used to store the geometry of the mesh. + The mapping will be approximated by an interpolation polynomial + of the specified degree for each tree. + The default of `1` creates an uncurved geometry. Use a higher value if the mapping + will curve the imported uncurved mesh. +- `RealT::Type`: The type that should be used for coordinates. +- `initial_refinement_level::Integer`: Refine the mesh uniformly to this level before the simulation starts. +- `boundary_symbols::Vector{Symbol}`: A vector of symbols that correspond to the boundary names in the `meshfile`. + If `nothing` is passed then all boundaries are named `:all`. +""" +function T8codeMesh(meshfile::AbaqusFile{NDIMS}; + mapping = nothing, polydeg = 1, RealT = Float64, + initial_refinement_level = 0, + boundary_symbols = nothing) where NDIMS + # Prevent `t8code` from crashing Julia if the file doesn't exist. + @assert isfile(meshfile.path) + + # Read in the Header of the meshfile to determine which constructor is appropriate. + header = open(meshfile.path, "r") do io + readline(io) # *Header of the Abaqus file; discarded + readline(io) # Readin the actual header information + end + + # Check if the meshfile was generated using HOHQMesh. + if header == " File created by HOHQMesh" + # Mesh curvature and boundary naming is handled with additional information available in meshfile + connectivity, tree_node_coordinates, nodes, boundary_names = p4est_connectivity_from_hohqmesh_abaqus(meshfile.path, + initial_refinement_level, + NDIMS, + RealT) + # Apply user defined mapping. + map_node_coordinates!(tree_node_coordinates, mapping) + else + # Mesh curvature is handled directly by applying the mapping keyword argument. + connectivity, tree_node_coordinates, nodes, boundary_names = p4est_connectivity_from_standard_abaqus(meshfile.path, + mapping, + polydeg, + initial_refinement_level, + NDIMS, + RealT, + boundary_symbols) + end + + if typeof(connectivity) <: Ptr{p4est_connectivity} + cmesh = t8_cmesh_new_from_p4est(connectivity, mpi_comm(), 0) + elseif typeof(connectivity) <: Ptr{p8est_connectivity} + cmesh = t8_cmesh_new_from_p8est(connectivity, mpi_comm(), 0) + else + throw("`connectivity` is not of type `Ptr{p*est_connectivity}`.") + end + + do_face_ghost = mpi_isparallel() + scheme = t8_scheme_new_default_cxx() + forest = t8_forest_new_uniform(cmesh, scheme, initial_refinement_level, do_face_ghost, + mpi_comm()) + + return T8codeMesh{NDIMS}(forest, tree_node_coordinates, nodes, + boundary_names, "") +end + struct adapt_callback_passthrough adapt_callback::Function user_data::Any diff --git a/test/test_t8code_3d.jl b/test/test_t8code_3d.jl index 4232cf04094..300eaef66c8 100644 --- a/test/test_t8code_3d.jl +++ b/test/test_t8code_3d.jl @@ -202,7 +202,7 @@ mkdir(outdir) 3.3228975127030935e-13, 9.592326932761353e-13, ], - tspan=(0.0, 0.1)) + tspan=(0.0, 0.1), atol=5.0e-13,) # Ensure that we do not have excessive memory allocations # (e.g., from type instabilities) let From 0fed965b941a3ec5877d438cd7545ae8eeeb14b1 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Mon, 8 Apr 2024 13:05:31 +0200 Subject: [PATCH 03/81] Updated examples. --- .../elixir_advection_amr_unstructured_flag.jl | 11 +++-------- .../elixir_advection_unstructured_flag.jl | 11 +++-------- examples/t8code_2d_dgsem/elixir_euler_free_stream.jl | 7 +------ ...er_source_terms_nonconforming_unstructured_flag.jl | 7 +------ examples/t8code_2d_dgsem/elixir_mhd_rotor.jl | 7 +------ .../elixir_advection_amr_unstructured_curved.jl | 7 +------ .../elixir_advection_unstructured_curved.jl | 7 +------ examples/t8code_3d_dgsem/elixir_euler_ec.jl | 7 +------ examples/t8code_3d_dgsem/elixir_euler_free_stream.jl | 7 +------ .../elixir_euler_free_stream_extruded.jl | 7 +------ ..._source_terms_nonconforming_unstructured_curved.jl | 7 +------ 11 files changed, 15 insertions(+), 70 deletions(-) diff --git a/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl b/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl index 0923e328487..6f43989c56b 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl @@ -33,14 +33,9 @@ mapping_flag = Trixi.transfinite_mapping(faces) mesh_file = Trixi.download("https://gist.githubusercontent.com/efaulhaber/63ff2ea224409e55ee8423b3a33e316a/raw/7db58af7446d1479753ae718930741c47a3b79b7/square_unstructured_2.inp", joinpath(@__DIR__, "square_unstructured_2.inp")) -# INP mesh files are only support by p4est. Hence, we -# create a p4est connecvity object first from which -# we can create a t8code mesh. -conn = Trixi.read_inp_p4est(mesh_file, Val(2)) - -mesh = T8codeMesh(conn, polydeg = 3, - mapping = mapping_flag, - initial_refinement_level = 1) +mesh = T8codeMesh(mesh_file, 2; + mapping = mapping_flag, polydeg = 3, + initial_refinement_level = 1) semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition, solver, boundary_conditions = boundary_conditions) diff --git a/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl b/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl index ba8f1b59b80..025493a1708 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl @@ -30,14 +30,9 @@ mapping_flag = Trixi.transfinite_mapping(faces) mesh_file = Trixi.download("https://gist.githubusercontent.com/efaulhaber/63ff2ea224409e55ee8423b3a33e316a/raw/7db58af7446d1479753ae718930741c47a3b79b7/square_unstructured_2.inp", joinpath(@__DIR__, "square_unstructured_2.inp")) -# INP mesh files are only support by p4est. Hence, we -# create a p4est connecvity object first from which -# we can create a t8code mesh. -conn = Trixi.read_inp_p4est(mesh_file, Val(2)) - -mesh = T8codeMesh(conn, polydeg = 3, - mapping = mapping_flag, - initial_refinement_level = 2) +mesh = T8codeMesh(mesh_file, 2; + mapping = mapping_flag, polydeg = 3, + initial_refinement_level = 2) # A semidiscretization collects data structures and functions for the spatial discretization. semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition, solver, diff --git a/examples/t8code_2d_dgsem/elixir_euler_free_stream.jl b/examples/t8code_2d_dgsem/elixir_euler_free_stream.jl index 5e6c4193c50..d9d2c65d988 100644 --- a/examples/t8code_2d_dgsem/elixir_euler_free_stream.jl +++ b/examples/t8code_2d_dgsem/elixir_euler_free_stream.jl @@ -32,12 +32,7 @@ end mesh_file = Trixi.download("https://gist.githubusercontent.com/efaulhaber/a075f8ec39a67fa9fad8f6f84342cbca/raw/a7206a02ed3a5d3cadacd8d9694ac154f9151db7/square_unstructured_1.inp", joinpath(@__DIR__, "square_unstructured_1.inp")) -# INP mesh files are only support by p4est. Hence, we -# create a p4est connecvity object first from which -# we can create a t8code mesh. -conn = Trixi.read_inp_p4est(mesh_file, Val(2)) - -mesh = T8codeMesh(conn, polydeg = 3, +mesh = T8codeMesh(mesh_file, 2; polydeg = 3, mapping = mapping, initial_refinement_level = 1) diff --git a/examples/t8code_2d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_flag.jl b/examples/t8code_2d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_flag.jl index e496eb76729..48684071d4b 100644 --- a/examples/t8code_2d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_flag.jl @@ -32,12 +32,7 @@ mapping_flag = Trixi.transfinite_mapping(faces) mesh_file = Trixi.download("https://gist.githubusercontent.com/efaulhaber/63ff2ea224409e55ee8423b3a33e316a/raw/7db58af7446d1479753ae718930741c47a3b79b7/square_unstructured_2.inp", joinpath(@__DIR__, "square_unstructured_2.inp")) -# INP mesh files are only support by p4est. Hence, we -# create a p4est connecvity object first from which -# we can create a t8code mesh. -conn = Trixi.read_inp_p4est(mesh_file, Val(2)) - -mesh = T8codeMesh(conn, polydeg = 3, +mesh = T8codeMesh(mesh_file, 2; polydeg = 3, mapping = mapping_flag, initial_refinement_level = 1) diff --git a/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl b/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl index ff2e40ae607..592d5b15a85 100644 --- a/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl +++ b/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl @@ -70,12 +70,7 @@ end mesh_file = Trixi.download("https://gist.githubusercontent.com/efaulhaber/63ff2ea224409e55ee8423b3a33e316a/raw/7db58af7446d1479753ae718930741c47a3b79b7/square_unstructured_2.inp", joinpath(@__DIR__, "square_unstructured_2.inp")) -# INP mesh files are only support by p4est. Hence, we -# create a p4est connecvity object first from which -# we can create a t8code mesh. -conn = Trixi.read_inp_p4est(mesh_file, Val(2)) - -mesh = T8codeMesh(conn, polydeg = 4, +mesh = T8codeMesh(mesh_file, 2; polydeg = 4, mapping = mapping_twist, initial_refinement_level = 1) diff --git a/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl b/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl index e7c0f4b7318..1f9aa3449b0 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl @@ -50,12 +50,7 @@ end mesh_file = Trixi.download("https://gist.githubusercontent.com/efaulhaber/b8df0033798e4926dec515fc045e8c2c/raw/b9254cde1d1fb64b6acc8416bc5ccdd77a240227/cube_unstructured_2.inp", joinpath(@__DIR__, "cube_unstructured_2.inp")) -# INP mesh files are only support by p4est. Hence, we -# create a p4est connectivity object first from which -# we can create a t8code mesh. -conn = Trixi.read_inp_p4est(mesh_file, Val(3)) - -mesh = T8codeMesh(conn, polydeg = 2, +mesh = T8codeMesh(mesh_file, 3; polydeg = 2, mapping = mapping, initial_refinement_level = 1) diff --git a/examples/t8code_3d_dgsem/elixir_advection_unstructured_curved.jl b/examples/t8code_3d_dgsem/elixir_advection_unstructured_curved.jl index ee27ee117fe..fe6aa48e7d9 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_unstructured_curved.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_unstructured_curved.jl @@ -47,12 +47,7 @@ end mesh_file = Trixi.download("https://gist.githubusercontent.com/efaulhaber/d45c8ac1e248618885fa7cc31a50ab40/raw/37fba24890ab37cfa49c39eae98b44faf4502882/cube_unstructured_1.inp", joinpath(@__DIR__, "cube_unstructured_1.inp")) -# INP mesh files are only support by p4est. Hence, we -# create a p4est connectivity object first from which -# we can create a t8code mesh. -conn = Trixi.read_inp_p4est(mesh_file, Val(3)) - -mesh = T8codeMesh(conn, polydeg = 3, +mesh = T8codeMesh(mesh_file, 3; polydeg = 3, mapping = mapping, initial_refinement_level = 2) diff --git a/examples/t8code_3d_dgsem/elixir_euler_ec.jl b/examples/t8code_3d_dgsem/elixir_euler_ec.jl index b720bfcd375..e1e4d850a86 100644 --- a/examples/t8code_3d_dgsem/elixir_euler_ec.jl +++ b/examples/t8code_3d_dgsem/elixir_euler_ec.jl @@ -47,12 +47,7 @@ end mesh_file = Trixi.download("https://gist.githubusercontent.com/efaulhaber/b8df0033798e4926dec515fc045e8c2c/raw/b9254cde1d1fb64b6acc8416bc5ccdd77a240227/cube_unstructured_2.inp", joinpath(@__DIR__, "cube_unstructured_2.inp")) -# INP mesh files are only support by p4est. Hence, we -# create a p4est connectivity object first from which -# we can create a t8code mesh. -conn = Trixi.read_inp_p4est(mesh_file, Val(3)) - -mesh = T8codeMesh(conn, polydeg = 5, +mesh = T8codeMesh(mesh_file, 3; polydeg = 5, mapping = mapping, initial_refinement_level = 0) diff --git a/examples/t8code_3d_dgsem/elixir_euler_free_stream.jl b/examples/t8code_3d_dgsem/elixir_euler_free_stream.jl index b70a6091adf..882e3aebebe 100644 --- a/examples/t8code_3d_dgsem/elixir_euler_free_stream.jl +++ b/examples/t8code_3d_dgsem/elixir_euler_free_stream.jl @@ -48,12 +48,7 @@ end mesh_file = Trixi.download("https://gist.githubusercontent.com/efaulhaber/d45c8ac1e248618885fa7cc31a50ab40/raw/37fba24890ab37cfa49c39eae98b44faf4502882/cube_unstructured_1.inp", joinpath(@__DIR__, "cube_unstructured_1.inp")) -# INP mesh files are only support by p4est. Hence, we -# create a p4est connectivity object first from which -# we can create a t8code mesh. -conn = Trixi.read_inp_p4est(mesh_file, Val(3)) - -mesh = T8codeMesh(conn, polydeg = 2, +mesh = T8codeMesh(mesh_file, 3; polydeg = 2, mapping = mapping, initial_refinement_level = 0) diff --git a/examples/t8code_3d_dgsem/elixir_euler_free_stream_extruded.jl b/examples/t8code_3d_dgsem/elixir_euler_free_stream_extruded.jl index 6ae38d20b5a..777cccf7ad7 100644 --- a/examples/t8code_3d_dgsem/elixir_euler_free_stream_extruded.jl +++ b/examples/t8code_3d_dgsem/elixir_euler_free_stream_extruded.jl @@ -37,12 +37,7 @@ end mesh_file = Trixi.download("https://gist.githubusercontent.com/efaulhaber/b8df0033798e4926dec515fc045e8c2c/raw/b9254cde1d1fb64b6acc8416bc5ccdd77a240227/cube_unstructured_2.inp", joinpath(@__DIR__, "cube_unstructured_2.inp")) -# INP mesh files are only support by p4est. Hence, we -# create a p4est connecvity object first from which -# we can create a t8code mesh. -conn = Trixi.read_inp_p4est(mesh_file, Val(3)) - -mesh = T8codeMesh(conn, polydeg = 3, +mesh = T8codeMesh(mesh_file, 3; polydeg = 3, mapping = mapping, initial_refinement_level = 0) diff --git a/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_curved.jl b/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_curved.jl index 6856be36ea1..a06e7927dd0 100644 --- a/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_curved.jl +++ b/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_curved.jl @@ -50,13 +50,8 @@ end mesh_file = Trixi.download("https://gist.githubusercontent.com/efaulhaber/d45c8ac1e248618885fa7cc31a50ab40/raw/37fba24890ab37cfa49c39eae98b44faf4502882/cube_unstructured_1.inp", joinpath(@__DIR__, "cube_unstructured_1.inp")) -# INP mesh files are only support by p4est. Hence, we -# create a p4est connecvity object first from which -# we can create a t8code mesh. -conn = Trixi.read_inp_p4est(mesh_file, Val(3)) - # Mesh polydeg of 2 (half the solver polydeg) to ensure FSP (see above). -mesh = T8codeMesh(conn, polydeg = 2, +mesh = T8codeMesh(mesh_file, 3; polydeg = 2, mapping = mapping, initial_refinement_level = 0) From c50b434e8d9408c641acba3f63d39d4d701f9651 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Mon, 8 Apr 2024 13:18:30 +0200 Subject: [PATCH 04/81] Fixed typos. --- src/meshes/t8code_mesh.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 72f7cf337b1..69ddb9a0bf7 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -106,7 +106,7 @@ end Create a 'T8codeMesh'. # Arguments -- 'forest': Pointer to a commited forest. +- 'forest': Pointer to a committed forest. - 'boundary_names': List of boundary names. - 'polydeg::Integer': Polynomial degree used to store the geometry of the mesh. The mapping will be approximated by an interpolation polynomial @@ -325,7 +325,7 @@ function T8codeMesh(trees_per_dimension; polydeg = 1, end end - # Note, `p*est_connectivity_new_brick` convers a domain of `[0,nx] x [0,ny] x ....`. + # Note, `p*est_connectivity_new_brick` converts a domain of `[0,nx] x [0,ny] x ....`. # Hence, transform mesh coordinates to reference space [-1,1]^NDIMS before applying user defined mapping. mapping_(xyz...) = mapping((x * 2.0/tpd - 1.0 for (x,tpd) in zip(xyz, trees_per_dimension))...) From 5e186563293bdf3dbb9f883b81400c9a00a40f58 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Mon, 8 Apr 2024 13:20:44 +0200 Subject: [PATCH 05/81] Applied formatter. --- .../elixir_advection_amr_unstructured_flag.jl | 4 +- .../elixir_advection_unstructured_flag.jl | 4 +- src/auxiliary/auxiliary.jl | 5 +- src/meshes/p4est_mesh.jl | 25 ++++---- src/meshes/t8code_mesh.jl | 62 ++++++++++--------- 5 files changed, 53 insertions(+), 47 deletions(-) diff --git a/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl b/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl index 6f43989c56b..9138586cccf 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl @@ -34,8 +34,8 @@ mesh_file = Trixi.download("https://gist.githubusercontent.com/efaulhaber/63ff2e joinpath(@__DIR__, "square_unstructured_2.inp")) mesh = T8codeMesh(mesh_file, 2; - mapping = mapping_flag, polydeg = 3, - initial_refinement_level = 1) + mapping = mapping_flag, polydeg = 3, + initial_refinement_level = 1) semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition, solver, boundary_conditions = boundary_conditions) diff --git a/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl b/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl index 025493a1708..e512f328234 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl @@ -31,8 +31,8 @@ mesh_file = Trixi.download("https://gist.githubusercontent.com/efaulhaber/63ff2e joinpath(@__DIR__, "square_unstructured_2.inp")) mesh = T8codeMesh(mesh_file, 2; - mapping = mapping_flag, polydeg = 3, - initial_refinement_level = 2) + mapping = mapping_flag, polydeg = 3, + initial_refinement_level = 2) # A semidiscretization collects data structures and functions for the spatial discretization. semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition, solver, diff --git a/src/auxiliary/auxiliary.jl b/src/auxiliary/auxiliary.jl index 34e7ccbbfdb..f3c54249816 100644 --- a/src/auxiliary/auxiliary.jl +++ b/src/auxiliary/auxiliary.jl @@ -374,11 +374,10 @@ end abstract type MeshFile{NDIMS} end struct GmshFile{NDIMS} <: MeshFile{NDIMS} - path :: String + path::String end struct AbaqusFile{NDIMS} <: MeshFile{NDIMS} - path :: String + path::String end - end # @muladd diff --git a/src/meshes/p4est_mesh.jl b/src/meshes/p4est_mesh.jl index 61713867e0f..6bb98196231 100644 --- a/src/meshes/p4est_mesh.jl +++ b/src/meshes/p4est_mesh.jl @@ -392,9 +392,9 @@ end function p4est_mesh_from_hohqmesh_abaqus(meshfile, initial_refinement_level, n_dimensions, RealT) connectivity, tree_node_coordinates, nodes, boundary_names = p4est_connectivity_from_hohqmesh_abaqus(meshfile, - initial_refinement_level, - n_dimensions, - RealT) + initial_refinement_level, + n_dimensions, + RealT) p4est = new_p4est(connectivity, initial_refinement_level) @@ -407,12 +407,12 @@ function p4est_mesh_from_standard_abaqus(meshfile, mapping, polydeg, initial_refinement_level, n_dimensions, RealT, boundary_symbols) connectivity, tree_node_coordinates, nodes, boundary_names = p4est_connectivity_from_standard_abaqus(meshfile, - mapping, - polydeg, - initial_refinement_level, - n_dimensions, - RealT, - boundary_symbols) + mapping, + polydeg, + initial_refinement_level, + n_dimensions, + RealT, + boundary_symbols) p4est = new_p4est(connectivity, initial_refinement_level) @@ -424,7 +424,7 @@ end # the boundary names on each tree are provided by the `meshfile` created by # [`HOHQMesh.jl`](https://github.com/trixi-framework/HOHQMesh.jl). function p4est_connectivity_from_hohqmesh_abaqus(meshfile, initial_refinement_level, - n_dimensions, RealT) + n_dimensions, RealT) # Create the mesh connectivity using `p4est` connectivity = read_inp_p4est(meshfile, Val(n_dimensions)) connectivity_pw = PointerWrapper(connectivity) @@ -480,8 +480,9 @@ end # the `mapping` passed to this function using polynomial interpolants of degree `polydeg`. All boundary # names are given the name `:all`. function p4est_connectivity_from_standard_abaqus(meshfile, mapping, polydeg, - initial_refinement_level, n_dimensions, RealT, - boundary_symbols) + initial_refinement_level, n_dimensions, + RealT, + boundary_symbols) # Create the mesh connectivity using `p4est` connectivity = read_inp_p4est(meshfile, Val(n_dimensions)) connectivity_pw = PointerWrapper(connectivity) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 69ddb9a0bf7..be76fb3f56b 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -26,7 +26,7 @@ mutable struct T8codeMesh{NDIMS, RealT <: Real, IsParallel, NDIMSP2, NNODES} <: nmpiinterfaces :: Int nmpimortars :: Int - function T8codeMesh{NDIMS}(forest :: Ptr{t8_forest}, tree_node_coordinates, nodes, + function T8codeMesh{NDIMS}(forest::Ptr{t8_forest}, tree_node_coordinates, nodes, boundary_names, current_filename) where {NDIMS} is_parallel = mpi_isparallel() ? True() : False() @@ -114,7 +114,8 @@ Create a 'T8codeMesh'. - `mapping`: a function of `NDIMS` variables to describe the mapping that transforms the imported mesh to the physical domain. Use `nothing` for the identity map. """ -function T8codeMesh{NDIMS, RealT}(forest :: Ptr{t8_forest}, boundary_names; polydeg = 1, mapping = nothing) where {NDIMS, RealT} +function T8codeMesh{NDIMS, RealT}(forest::Ptr{t8_forest}, boundary_names; polydeg = 1, + mapping = nothing) where {NDIMS, RealT} # In t8code reference space is [0,1]. basis = LobattoLegendreBasis(RealT, polydeg) nodes = 0.5 .* (basis.nodes .+ 1.0) @@ -135,7 +136,7 @@ function T8codeMesh{NDIMS, RealT}(forest :: Ptr{t8_forest}, boundary_names; poly # Testing for negative element volumes. verts = zeros(3, num_corners) for itree in 1:num_trees - veptr = t8_cmesh_get_tree_vertices(cmesh, itree-1) + veptr = t8_cmesh_get_tree_vertices(cmesh, itree - 1) # Note, `verts = unsafe_wrap(Array, veptr, (3, 1 << NDIMS))` # sometimes does not work since `veptr` is not necessarily properly @@ -164,7 +165,8 @@ function T8codeMesh{NDIMS, RealT}(forest :: Ptr{t8_forest}, boundary_names; poly coords_ref[1] = nodes[i] coords_ref[2] = nodes[j] coords_ref[3] = 0.0 - t8_geometry_evaluate(cmesh, itree-1, coords_ref, 1, @view(tree_node_coordinates[:, i, j, itree])) + t8_geometry_evaluate(cmesh, itree - 1, coords_ref, 1, + @view(tree_node_coordinates[:, i, j, itree])) end end @@ -174,7 +176,7 @@ function T8codeMesh{NDIMS, RealT}(forest :: Ptr{t8_forest}, boundary_names; poly # Testing for negative element volumes. verts = zeros(3, num_corners) for itree in 1:num_trees - veptr = t8_cmesh_get_tree_vertices(cmesh, itree-1) + veptr = t8_cmesh_get_tree_vertices(cmesh, itree - 1) # Note, `verts = unsafe_wrap(Array, veptr, (3, 1 << NDIMS))` # sometimes does not work since `veptr` is not necessarily properly @@ -204,7 +206,8 @@ function T8codeMesh{NDIMS, RealT}(forest :: Ptr{t8_forest}, boundary_names; poly coords_ref[1] = nodes[i] coords_ref[2] = nodes[j] coords_ref[3] = nodes[k] - t8_geometry_evaluate(cmesh, itree-1, coords_ref, 1, @view(tree_node_coordinates[:, i, j, k, itree])) + t8_geometry_evaluate(cmesh, itree - 1, coords_ref, 1, + @view(tree_node_coordinates[:, i, j, k, itree])) end end end @@ -327,9 +330,11 @@ function T8codeMesh(trees_per_dimension; polydeg = 1, # Note, `p*est_connectivity_new_brick` converts a domain of `[0,nx] x [0,ny] x ....`. # Hence, transform mesh coordinates to reference space [-1,1]^NDIMS before applying user defined mapping. - mapping_(xyz...) = mapping((x * 2.0/tpd - 1.0 for (x,tpd) in zip(xyz, trees_per_dimension))...) + mapping_(xyz...) = mapping((x * 2.0 / tpd - 1.0 for (x, tpd) in zip(xyz, + trees_per_dimension))...) - return T8codeMesh{NDIMS, RealT}(forest, boundary_names; polydeg = polydeg, mapping = mapping_) + return T8codeMesh{NDIMS, RealT}(forest, boundary_names; polydeg = polydeg, + mapping = mapping_) end """ @@ -370,7 +375,8 @@ function T8codeMesh(cmesh::Ptr{t8_cmesh}; # There's no simple and generic way to distinguish boundaries, yet. Name all of them :all. boundary_names = fill(:all, 2 * NDIMS, t8_cmesh_get_num_trees(cmesh)) - return T8codeMesh{NDIMS, RealT}(forest, boundary_names; polydeg = polydeg, mapping = mapping) + return T8codeMesh{NDIMS, RealT}(forest, boundary_names; polydeg = polydeg, + mapping = mapping) end """ @@ -452,11 +458,11 @@ function T8codeMesh(filepath::String, NDIMS; kwargs...) file_extension = lowercase(meshfile_suffix) if file_extension == ".msh" - return T8codeMesh(GmshFile{NDIMS}(filepath); kwargs...) + return T8codeMesh(GmshFile{NDIMS}(filepath); kwargs...) end if file_extension == ".inp" - return T8codeMesh(AbaqusFile{NDIMS}(filepath); kwargs...) + return T8codeMesh(AbaqusFile{NDIMS}(filepath); kwargs...) end throw("Unknown file extension: " * file_extension) @@ -482,7 +488,7 @@ mesh from a Gmsh mesh file (`.msh`). - `RealT::Type`: The type that should be used for coordinates. - `initial_refinement_level::Integer`: Refine the mesh uniformly to this level before the simulation starts. """ -function T8codeMesh(meshfile::GmshFile{NDIMS}; kwargs...) where NDIMS +function T8codeMesh(meshfile::GmshFile{NDIMS}; kwargs...) where {NDIMS} # Prevent `t8code` from crashing Julia if the file doesn't exist. @assert isfile(meshfile.path) @@ -555,9 +561,9 @@ For example, if a two-dimensional base mesh contains 25 elements then setting If `nothing` is passed then all boundaries are named `:all`. """ function T8codeMesh(meshfile::AbaqusFile{NDIMS}; - mapping = nothing, polydeg = 1, RealT = Float64, - initial_refinement_level = 0, - boundary_symbols = nothing) where NDIMS + mapping = nothing, polydeg = 1, RealT = Float64, + initial_refinement_level = 0, + boundary_symbols = nothing) where {NDIMS} # Prevent `t8code` from crashing Julia if the file doesn't exist. @assert isfile(meshfile.path) @@ -571,30 +577,30 @@ function T8codeMesh(meshfile::AbaqusFile{NDIMS}; if header == " File created by HOHQMesh" # Mesh curvature and boundary naming is handled with additional information available in meshfile connectivity, tree_node_coordinates, nodes, boundary_names = p4est_connectivity_from_hohqmesh_abaqus(meshfile.path, - initial_refinement_level, - NDIMS, - RealT) + initial_refinement_level, + NDIMS, + RealT) # Apply user defined mapping. map_node_coordinates!(tree_node_coordinates, mapping) else # Mesh curvature is handled directly by applying the mapping keyword argument. connectivity, tree_node_coordinates, nodes, boundary_names = p4est_connectivity_from_standard_abaqus(meshfile.path, - mapping, - polydeg, - initial_refinement_level, - NDIMS, - RealT, - boundary_symbols) + mapping, + polydeg, + initial_refinement_level, + NDIMS, + RealT, + boundary_symbols) end if typeof(connectivity) <: Ptr{p4est_connectivity} - cmesh = t8_cmesh_new_from_p4est(connectivity, mpi_comm(), 0) + cmesh = t8_cmesh_new_from_p4est(connectivity, mpi_comm(), 0) elseif typeof(connectivity) <: Ptr{p8est_connectivity} - cmesh = t8_cmesh_new_from_p8est(connectivity, mpi_comm(), 0) + cmesh = t8_cmesh_new_from_p8est(connectivity, mpi_comm(), 0) else - throw("`connectivity` is not of type `Ptr{p*est_connectivity}`.") + throw("`connectivity` is not of type `Ptr{p*est_connectivity}`.") end - + do_face_ghost = mpi_isparallel() scheme = t8_scheme_new_default_cxx() forest = t8_forest_new_uniform(cmesh, scheme, initial_refinement_level, do_face_ghost, From 3bb6292e01f50ad8bb2a19dc9a807bb8f5015501 Mon Sep 17 00:00:00 2001 From: Benedict Geihe Date: Wed, 13 Mar 2024 17:11:51 +0100 Subject: [PATCH 06/81] cubed sphere test case, copied from p4est --- .../elixir_advection_cubed_sphere.jl | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 examples/t8code_3d_dgsem/elixir_advection_cubed_sphere.jl diff --git a/examples/t8code_3d_dgsem/elixir_advection_cubed_sphere.jl b/examples/t8code_3d_dgsem/elixir_advection_cubed_sphere.jl new file mode 100644 index 00000000000..9f9876d2753 --- /dev/null +++ b/examples/t8code_3d_dgsem/elixir_advection_cubed_sphere.jl @@ -0,0 +1,61 @@ + +using OrdinaryDiffEq +using Trixi + +############################################################################### +# semidiscretization of the linear advection equation + +advection_velocity = (0.2, -0.7, 0.5) +equations = LinearScalarAdvectionEquation3D(advection_velocity) + +# Create DG solver with polynomial degree = 3 and (local) Lax-Friedrichs/Rusanov flux as surface flux +solver = DGSEM(polydeg = 3, surface_flux = flux_lax_friedrichs) + +initial_condition = initial_condition_convergence_test + +boundary_condition = BoundaryConditionDirichlet(initial_condition) +boundary_conditions = Dict(:inside => boundary_condition, + :outside => boundary_condition) + +mesh = Trixi.T8codeMeshCubedSphere(5, 3, 0.5, 0.5; + polydeg = 3, initial_refinement_level = 0) + +# A semidiscretization collects data structures and functions for the spatial discretization +semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition, solver, + boundary_conditions = boundary_conditions) + +############################################################################### +# ODE solvers, callbacks etc. + +# Create ODE problem with time span from 0.0 to 1.0 +tspan = (0.0, 1.0) +ode = semidiscretize(semi, tspan) + +# At the beginning of the main loop, the SummaryCallback prints a summary of the simulation setup +# and resets the timers +summary_callback = SummaryCallback() + +# The AnalysisCallback allows to analyse the solution in regular intervals and prints the results +analysis_callback = AnalysisCallback(semi, interval = 100) + +# The SaveSolutionCallback allows to save the solution to a file in regular intervals +save_solution = SaveSolutionCallback(interval = 100, + solution_variables = cons2prim) + +# The StepsizeCallback handles the re-calculation of the maximum Δt after each time step +stepsize_callback = StepsizeCallback(cfl = 1.2) + +# Create a CallbackSet to collect all callbacks such that they can be passed to the ODE solver +callbacks = CallbackSet(summary_callback, analysis_callback, save_solution, + stepsize_callback) + +############################################################################### +# run the simulation + +# OrdinaryDiffEq's `solve` method evolves the solution in time and executes the passed callbacks +sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), + dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback + save_everystep = false, callback = callbacks); + +# Print the timer summary +summary_callback() From abbf702d5e426670fc63cea5c7229f59d1b1bbe5 Mon Sep 17 00:00:00 2001 From: Benedict Geihe Date: Wed, 27 Mar 2024 10:28:29 +0100 Subject: [PATCH 07/81] add baroclinic instability (copy of p4est) --- .../elixir_euler_baroclinic_instability.jl | 299 ++++++++++++++++++ 1 file changed, 299 insertions(+) create mode 100644 examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl diff --git a/examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl b/examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl new file mode 100644 index 00000000000..128f1e38a5b --- /dev/null +++ b/examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl @@ -0,0 +1,299 @@ +# An idealized baroclinic instability test case +# For optimal results consider increasing the resolution to 16x16x8 trees per cube face. +# +# Note that this elixir can take several hours to run. +# Using 24 threads of an AMD Ryzen Threadripper 3990X (more threads don't speed it up further) +# and `check-bounds=no`, this elixirs takes about one hour to run. +# With 16x16x8 trees per cube face on the same machine, it takes about 28 hours. +# +# References: +# - Paul A. Ullrich, Thomas Melvin, Christiane Jablonowski, Andrew Staniforth (2013) +# A proposed baroclinic wave test case for deep- and shallow-atmosphere dynamical cores +# https://doi.org/10.1002/qj.2241 + +using OrdinaryDiffEq +using Trixi +using LinearAlgebra + +############################################################################### +# Setup for the baroclinic instability test +gamma = 1.4 +equations = CompressibleEulerEquations3D(gamma) + +# Initial condition for an idealized baroclinic instability test +# https://doi.org/10.1002/qj.2241, Section 3.2 and Appendix A +function initial_condition_baroclinic_instability(x, t, + equations::CompressibleEulerEquations3D) + lon, lat, r = cartesian_to_sphere(x) + radius_earth = 6.371229e6 + # Make sure that the r is not smaller than radius_earth + z = max(r - radius_earth, 0.0) + + # Unperturbed basic state + rho, u, p = basic_state_baroclinic_instability_longitudinal_velocity(lon, lat, z) + + # Stream function type perturbation + u_perturbation, v_perturbation = perturbation_stream_function(lon, lat, z) + + u += u_perturbation + v = v_perturbation + + # Convert spherical velocity to Cartesian + v1 = -sin(lon) * u - sin(lat) * cos(lon) * v + v2 = cos(lon) * u - sin(lat) * sin(lon) * v + v3 = cos(lat) * v + + return prim2cons(SVector(rho, v1, v2, v3, p), equations) +end + +# Steady state for RHS correction below +function steady_state_baroclinic_instability(x, t, equations::CompressibleEulerEquations3D) + lon, lat, r = cartesian_to_sphere(x) + radius_earth = 6.371229e6 + # Make sure that the r is not smaller than radius_earth + z = max(r - radius_earth, 0.0) + + # Unperturbed basic state + rho, u, p = basic_state_baroclinic_instability_longitudinal_velocity(lon, lat, z) + + # Convert spherical velocity to Cartesian + v1 = -sin(lon) * u + v2 = cos(lon) * u + v3 = 0.0 + + return prim2cons(SVector(rho, v1, v2, v3, p), equations) +end + +function cartesian_to_sphere(x) + r = norm(x) + lambda = atan(x[2], x[1]) + if lambda < 0 + lambda += 2 * pi + end + phi = asin(x[3] / r) + + return lambda, phi, r +end + +# Unperturbed balanced steady-state. +# Returns primitive variables with only the velocity in longitudinal direction (rho, u, p). +# The other velocity components are zero. +function basic_state_baroclinic_instability_longitudinal_velocity(lon, lat, z) + # Parameters from Table 1 in the paper + # Corresponding names in the paper are commented + radius_earth = 6.371229e6 # a + half_width_parameter = 2 # b + gravitational_acceleration = 9.80616 # g + k = 3 # k + surface_pressure = 1e5 # p₀ + gas_constant = 287 # R + surface_equatorial_temperature = 310.0 # T₀ᴱ + surface_polar_temperature = 240.0 # T₀ᴾ + lapse_rate = 0.005 # Γ + angular_velocity = 7.29212e-5 # Ω + + # Distance to the center of the Earth + r = z + radius_earth + + # In the paper: T₀ + temperature0 = 0.5 * (surface_equatorial_temperature + surface_polar_temperature) + # In the paper: A, B, C, H + const_a = 1 / lapse_rate + const_b = (temperature0 - surface_polar_temperature) / + (temperature0 * surface_polar_temperature) + const_c = 0.5 * (k + 2) * (surface_equatorial_temperature - surface_polar_temperature) / + (surface_equatorial_temperature * surface_polar_temperature) + const_h = gas_constant * temperature0 / gravitational_acceleration + + # In the paper: (r - a) / bH + scaled_z = z / (half_width_parameter * const_h) + + # Temporary variables + temp1 = exp(lapse_rate / temperature0 * z) + temp2 = exp(-scaled_z^2) + + # In the paper: ̃τ₁, ̃τ₂ + tau1 = const_a * lapse_rate / temperature0 * temp1 + + const_b * (1 - 2 * scaled_z^2) * temp2 + tau2 = const_c * (1 - 2 * scaled_z^2) * temp2 + + # In the paper: ∫τ₁(r') dr', ∫τ₂(r') dr' + inttau1 = const_a * (temp1 - 1) + const_b * z * temp2 + inttau2 = const_c * z * temp2 + + # Temporary variables + temp3 = r / radius_earth * cos(lat) + temp4 = temp3^k - k / (k + 2) * temp3^(k + 2) + + # In the paper: T + temperature = 1 / ((r / radius_earth)^2 * (tau1 - tau2 * temp4)) + + # In the paper: U, u (zonal wind, first component of spherical velocity) + big_u = gravitational_acceleration / radius_earth * k * temperature * inttau2 * + (temp3^(k - 1) - temp3^(k + 1)) + temp5 = radius_earth * cos(lat) + u = -angular_velocity * temp5 + sqrt(angular_velocity^2 * temp5^2 + temp5 * big_u) + + # Hydrostatic pressure + p = surface_pressure * + exp(-gravitational_acceleration / gas_constant * (inttau1 - inttau2 * temp4)) + + # Density (via ideal gas law) + rho = p / (gas_constant * temperature) + + return rho, u, p +end + +# Perturbation as in Equations 25 and 26 of the paper (analytical derivative) +function perturbation_stream_function(lon, lat, z) + # Parameters from Table 1 in the paper + # Corresponding names in the paper are commented + perturbation_radius = 1 / 6 # d₀ / a + perturbed_wind_amplitude = 1.0 # Vₚ + perturbation_lon = pi / 9 # Longitude of perturbation location + perturbation_lat = 2 * pi / 9 # Latitude of perturbation location + pertz = 15000 # Perturbation height cap + + # Great circle distance (d in the paper) divided by a (radius of the Earth) + # because we never actually need d without dividing by a + great_circle_distance_by_a = acos(sin(perturbation_lat) * sin(lat) + + cos(perturbation_lat) * cos(lat) * + cos(lon - perturbation_lon)) + + # In the first case, the vertical taper function is per definition zero. + # In the second case, the stream function is per definition zero. + if z > pertz || great_circle_distance_by_a > perturbation_radius + return 0.0, 0.0 + end + + # Vertical tapering of stream function + perttaper = 1.0 - 3 * z^2 / pertz^2 + 2 * z^3 / pertz^3 + + # sin/cos(pi * d / (2 * d_0)) in the paper + sin_, cos_ = sincos(0.5 * pi * great_circle_distance_by_a / perturbation_radius) + + # Common factor for both u and v + factor = 16 / (3 * sqrt(3)) * perturbed_wind_amplitude * perttaper * cos_^3 * sin_ + + u_perturbation = -factor * (-sin(perturbation_lat) * cos(lat) + + cos(perturbation_lat) * sin(lat) * cos(lon - perturbation_lon)) / + sin(great_circle_distance_by_a) + + v_perturbation = factor * cos(perturbation_lat) * sin(lon - perturbation_lon) / + sin(great_circle_distance_by_a) + + return u_perturbation, v_perturbation +end + +@inline function source_terms_baroclinic_instability(u, x, t, + equations::CompressibleEulerEquations3D) + radius_earth = 6.371229e6 # a + gravitational_acceleration = 9.80616 # g + angular_velocity = 7.29212e-5 # Ω + + r = norm(x) + # Make sure that r is not smaller than radius_earth + z = max(r - radius_earth, 0.0) + r = z + radius_earth + + du1 = zero(eltype(u)) + + # Gravity term + temp = -gravitational_acceleration * radius_earth^2 / r^3 + du2 = temp * u[1] * x[1] + du3 = temp * u[1] * x[2] + du4 = temp * u[1] * x[3] + du5 = temp * (u[2] * x[1] + u[3] * x[2] + u[4] * x[3]) + + # Coriolis term, -2Ω × ρv = -2 * angular_velocity * (0, 0, 1) × u[2:4] + du2 -= -2 * angular_velocity * u[3] + du3 -= 2 * angular_velocity * u[2] + + return SVector(du1, du2, du3, du4, du5) +end + +############################################################################### +# Start of the actual elixir, semidiscretization of the problem + +initial_condition = initial_condition_baroclinic_instability + +boundary_conditions = Dict(:inside => boundary_condition_slip_wall, + :outside => boundary_condition_slip_wall) + +# This is a good estimate for the speed of sound in this example. +# Other values between 300 and 400 should work as well. +surface_flux = FluxLMARS(340) +volume_flux = flux_kennedy_gruber +solver = DGSEM(polydeg = 5, surface_flux = surface_flux, + volume_integral = VolumeIntegralFluxDifferencing(volume_flux)) + +# For optimal results, use (16, 8) here +trees_per_cube_face = (8, 4) +mesh = Trixi.T8codeMeshCubedSphere(trees_per_cube_face..., 6.371229e6, 30000.0, + polydeg = 5, initial_refinement_level = 0) + +semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition, solver, + source_terms = source_terms_baroclinic_instability, + boundary_conditions = boundary_conditions) + +############################################################################### +# ODE solvers, callbacks etc. + +tspan = (0.0, 10 * 24 * 60 * 60.0) # time in seconds for 10 days + +# Save RHS of the steady state and subtract it in every RHS evaluation. +# This trick preserves the steady state exactly (to machine rounding errors, of course). +# Otherwise, this elixir produces entirely unusable results for a resolution of 8x8x4 cells +# per cube face with a polydeg of 3. +# With this trick, even the polydeg 3 simulation produces usable (although badly resolved) results, +# and most of the grid imprinting in higher polydeg simulation is eliminated. +# +# See https://github.com/trixi-framework/Trixi.jl/issues/980 for more information. +u_steady_state = compute_coefficients(steady_state_baroclinic_instability, tspan[1], semi) +# Use a `let` block for performance (otherwise du_steady_state will be a global variable) +let du_steady_state = similar(u_steady_state) + # Save RHS of the steady state + Trixi.rhs!(du_steady_state, u_steady_state, semi, tspan[1]) + + global function corrected_rhs!(du, u, semi, t) + # Normal RHS evaluation + Trixi.rhs!(du, u, semi, t) + # Correct by subtracting the steady-state RHS + Trixi.@trixi_timeit Trixi.timer() "rhs correction" begin + # Use Trixi.@threaded for threaded performance + Trixi.@threaded for i in eachindex(du) + du[i] -= du_steady_state[i] + end + end + end +end +u0 = compute_coefficients(tspan[1], semi) +ode = ODEProblem(corrected_rhs!, u0, tspan, semi) + +summary_callback = SummaryCallback() + +analysis_interval = 5000 +analysis_callback = AnalysisCallback(semi, interval = analysis_interval) + +alive_callback = AliveCallback(analysis_interval = analysis_interval) + +#save_solution = SaveSolutionCallback(interval = 5000, +# save_initial_solution = true, +# save_final_solution = true, +# solution_variables = cons2prim) + +callbacks = CallbackSet(summary_callback, + analysis_callback, + alive_callback) +# , save_solution) + +############################################################################### +# run the simulation + +# Use a Runge-Kutta method with automatic (error based) time step size control +# Enable threading of the RK method for better performance on multiple threads +sol = solve(ode, RDPK3SpFSAL49(thread = OrdinaryDiffEq.True()); abstol = 1.0e-6, + reltol = 1.0e-6, + ode_default_options()..., callback = callbacks); + +summary_callback() # print the timer summary From 5fa48e83ae95e9197b0d4faddc327bda95c91a4c Mon Sep 17 00:00:00 2001 From: Benedict Geihe Date: Tue, 9 Apr 2024 13:13:08 +0200 Subject: [PATCH 08/81] add cubed sphere constructor --- src/meshes/t8code_mesh.jl | 45 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index be76fb3f56b..007da861f5b 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -610,6 +610,51 @@ function T8codeMesh(meshfile::AbaqusFile{NDIMS}; boundary_names, "") end +""" +T8codeMeshCubedSphere(trees_per_face_dimension, layers, inner_radius, thickness; + polydeg, RealT=Float64, initial_refinement_level=0) + +Construct a cubed spherical shell of given inner radius and thickness as `T8codeMesh` with +`6 * trees_per_face_dimension^2 * layers` trees. The mesh will have two boundaries, +`:inside` and `:outside`. + +# Arguments +- `trees_per_face_dimension::Integer`: the number of trees in the first two local + dimensions of each face. +- `layers::Integer`: the number of trees in the third local dimension of each face, i.e., + the number of layers of the shell. +- `inner_radius::Float64`: Radius of the inner side of the shell. +- `thickness::Float64`: Thickness of the shell. The outer radius will be + `inner_radius + thickness`. +- `polydeg::Integer`: polynomial degree used to store the geometry of the mesh. + The mapping will be approximated by an interpolation polynomial + of the specified degree for each tree. +- `RealT::Type`: the type that should be used for coordinates. +- `initial_refinement_level::Integer`: refine the mesh uniformly to this level before the + simulation starts. +""" +function T8codeMeshCubedSphere(trees_per_face_dimension, layers, inner_radius, thickness; + polydeg, RealT = Float64, initial_refinement_level = 0) + NDIMS = 3 + cmesh = t8_cmesh_new_cubed_spherical_shell(inner_radius, thickness, + trees_per_face_dimension, layers, mpi_comm()) + do_face_ghost = mpi_isparallel() + scheme = t8_scheme_new_default_cxx() + forest = t8_forest_new_uniform(cmesh, scheme, initial_refinement_level, do_face_ghost, + mpi_comm()) + + num_trees = t8_cmesh_get_num_trees(cmesh) + # TODO: Init?! + boundary_names = fill(Symbol("---"), 2 * NDIMS, num_trees) + for itree in 1:num_trees + # TODO: z-direction == radial direction in each tree? + boundary_names[5, itree] = :inside + boundary_names[6, itree] = :outside + end + + return T8codeMesh{NDIMS, RealT}(forest, boundary_names; polydeg = polydeg) +end + struct adapt_callback_passthrough adapt_callback::Function user_data::Any From cd27998913e96218e37844b1622b06d521b3c705 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Fri, 19 Apr 2024 09:56:15 +0200 Subject: [PATCH 09/81] Fix indentation. --- src/solvers/dgsem_t8code/containers_2d.jl | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/solvers/dgsem_t8code/containers_2d.jl b/src/solvers/dgsem_t8code/containers_2d.jl index ce525bfdf65..1fdee28fc72 100644 --- a/src/solvers/dgsem_t8code/containers_2d.jl +++ b/src/solvers/dgsem_t8code/containers_2d.jl @@ -41,22 +41,14 @@ function calc_node_coordinates!(node_coordinates, t8_element_vertex_reference_coords(eclass_scheme, element, 0, pointer(element_coords)) - nodes_out_x = 2 * - (element_length * 1 / 2 * (nodes .+ 1) .+ element_coords[1]) .- - 1 - nodes_out_y = 2 * - (element_length * 1 / 2 * (nodes .+ 1) .+ element_coords[2]) .- - 1 + nodes_out_x = 2 * (element_length * 1 / 2 * (nodes .+ 1) .+ element_coords[1]) .- 1 + nodes_out_y = 2 * (element_length * 1 / 2 * (nodes .+ 1) .+ element_coords[2]) .- 1 - polynomial_interpolation_matrix!(matrix1, mesh.nodes, nodes_out_x, - baryweights_in) - polynomial_interpolation_matrix!(matrix2, mesh.nodes, nodes_out_y, - baryweights_in) + polynomial_interpolation_matrix!(matrix1, mesh.nodes, nodes_out_x, baryweights_in) + polynomial_interpolation_matrix!(matrix2, mesh.nodes, nodes_out_y, baryweights_in) multiply_dimensionwise!(view(node_coordinates, :, :, :, current_index += 1), - matrix1, matrix2, - view(mesh.tree_node_coordinates, :, :, :, - global_itree + 1), + matrix1, matrix2, view(mesh.tree_node_coordinates, :, :, :, global_itree + 1), tmp1) end end From f6653b0fc95adfab7a08c12ceab048bcf82f5b01 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Fri, 19 Apr 2024 09:58:15 +0200 Subject: [PATCH 10/81] Fix indentation. --- src/solvers/dgsem_t8code/containers_3d.jl | 32 +++++++---------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/src/solvers/dgsem_t8code/containers_3d.jl b/src/solvers/dgsem_t8code/containers_3d.jl index 4d56bc734aa..fddb2d01478 100644 --- a/src/solvers/dgsem_t8code/containers_3d.jl +++ b/src/solvers/dgsem_t8code/containers_3d.jl @@ -43,29 +43,17 @@ function calc_node_coordinates!(node_coordinates, t8_element_vertex_reference_coords(eclass_scheme, element, 0, pointer(element_coords)) - nodes_out_x = (2 * - (element_length * 0.5 * (nodes .+ 1) .+ element_coords[1]) .- - 1) - nodes_out_y = (2 * - (element_length * 0.5 * (nodes .+ 1) .+ element_coords[2]) .- - 1) - nodes_out_z = (2 * - (element_length * 0.5 * (nodes .+ 1) .+ element_coords[3]) .- - 1) - - polynomial_interpolation_matrix!(matrix1, mesh.nodes, nodes_out_x, - baryweights_in) - polynomial_interpolation_matrix!(matrix2, mesh.nodes, nodes_out_y, - baryweights_in) - polynomial_interpolation_matrix!(matrix3, mesh.nodes, nodes_out_z, - baryweights_in) - - multiply_dimensionwise!(view(node_coordinates, :, :, :, :, - current_index += 1), + nodes_out_x = (2 * (element_length * 0.5 * (nodes .+ 1) .+ element_coords[1]) .- 1) + nodes_out_y = (2 * (element_length * 0.5 * (nodes .+ 1) .+ element_coords[2]) .- 1) + nodes_out_z = (2 * (element_length * 0.5 * (nodes .+ 1) .+ element_coords[3]) .- 1) + + polynomial_interpolation_matrix!(matrix1, mesh.nodes, nodes_out_x, baryweights_in) + polynomial_interpolation_matrix!(matrix2, mesh.nodes, nodes_out_y, baryweights_in) + polynomial_interpolation_matrix!(matrix3, mesh.nodes, nodes_out_z, baryweights_in) + + multiply_dimensionwise!(view(node_coordinates, :, :, :, :, current_index += 1), matrix1, matrix2, matrix3, - view(mesh.tree_node_coordinates, :, :, :, :, - global_itree + 1), - tmp1) + view(mesh.tree_node_coordinates, :, :, :, :, global_itree + 1), tmp1) end end From aa4d0d4fecb7eccf3d64a43d0ca413d83136f660 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Fri, 19 Apr 2024 10:36:17 +0200 Subject: [PATCH 11/81] Switching off formatter in two files. --- src/solvers/dgsem_t8code/containers_2d.jl | 1 + src/solvers/dgsem_t8code/containers_3d.jl | 1 + 2 files changed, 2 insertions(+) diff --git a/src/solvers/dgsem_t8code/containers_2d.jl b/src/solvers/dgsem_t8code/containers_2d.jl index 1fdee28fc72..104e5590a58 100644 --- a/src/solvers/dgsem_t8code/containers_2d.jl +++ b/src/solvers/dgsem_t8code/containers_2d.jl @@ -4,6 +4,7 @@ # See https://ranocha.de/blog/Optimizing_EC_Trixi for further details. @muladd begin #! format: noindent +#! format: off # Interpolate tree_node_coordinates to each quadrant at the specified nodes. function calc_node_coordinates!(node_coordinates, diff --git a/src/solvers/dgsem_t8code/containers_3d.jl b/src/solvers/dgsem_t8code/containers_3d.jl index fddb2d01478..e1e58fafd85 100644 --- a/src/solvers/dgsem_t8code/containers_3d.jl +++ b/src/solvers/dgsem_t8code/containers_3d.jl @@ -4,6 +4,7 @@ # See https://ranocha.de/blog/Optimizing_EC_Trixi for further details. @muladd begin #! format: noindent +#! format: off # Interpolate tree_node_coordinates to each quadrant at the specified nodes function calc_node_coordinates!(node_coordinates, From 40cbb8662403f28962134f542c7e0bd399364360 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Fri, 19 Apr 2024 10:43:39 +0200 Subject: [PATCH 12/81] Upgrading T8code.jl. --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 5e1a60b7723..450c7a0087b 100644 --- a/Project.toml +++ b/Project.toml @@ -92,7 +92,7 @@ StaticArrays = "1.5" StrideArrays = "0.1.26" StructArrays = "0.6.11" SummationByPartsOperators = "0.5.41" -T8code = "0.4.3, 0.5" +T8code = "0.6.0" TimerOutputs = "0.5.7" Triangulate = "2.2" TriplotBase = "0.1" From df77f82fb8a29a89656ab9109f717079b0c7205a Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Fri, 19 Apr 2024 11:34:22 +0200 Subject: [PATCH 13/81] Fixed examples. --- examples/t8code_3d_dgsem/elixir_advection_cubed_sphere.jl | 8 ++++---- .../elixir_euler_baroclinic_instability.jl | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/examples/t8code_3d_dgsem/elixir_advection_cubed_sphere.jl b/examples/t8code_3d_dgsem/elixir_advection_cubed_sphere.jl index 9f9876d2753..9163cadada8 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_cubed_sphere.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_cubed_sphere.jl @@ -38,15 +38,15 @@ summary_callback = SummaryCallback() # The AnalysisCallback allows to analyse the solution in regular intervals and prints the results analysis_callback = AnalysisCallback(semi, interval = 100) -# The SaveSolutionCallback allows to save the solution to a file in regular intervals -save_solution = SaveSolutionCallback(interval = 100, - solution_variables = cons2prim) +# # The SaveSolutionCallback allows to save the solution to a file in regular intervals +# save_solution = SaveSolutionCallback(interval = 100, +# solution_variables = cons2prim) # The StepsizeCallback handles the re-calculation of the maximum Δt after each time step stepsize_callback = StepsizeCallback(cfl = 1.2) # Create a CallbackSet to collect all callbacks such that they can be passed to the ODE solver -callbacks = CallbackSet(summary_callback, analysis_callback, save_solution, +callbacks = CallbackSet(summary_callback, analysis_callback, # save_solution, stepsize_callback) ############################################################################### diff --git a/examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl b/examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl index 128f1e38a5b..4f5b187885e 100644 --- a/examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl +++ b/examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl @@ -227,8 +227,7 @@ volume_flux = flux_kennedy_gruber solver = DGSEM(polydeg = 5, surface_flux = surface_flux, volume_integral = VolumeIntegralFluxDifferencing(volume_flux)) -# For optimal results, use (16, 8) here -trees_per_cube_face = (8, 4) +trees_per_cube_face = (4, 4) mesh = Trixi.T8codeMeshCubedSphere(trees_per_cube_face..., 6.371229e6, 30000.0, polydeg = 5, initial_refinement_level = 0) From b80f890049b2a5e81b7d82a2fe58f5f2f4fc167c Mon Sep 17 00:00:00 2001 From: Benedict Geihe Date: Mon, 22 Apr 2024 11:21:26 +0200 Subject: [PATCH 14/81] stress different meaning of first argument it refers to level of refinement in lat lon direction, not number of tree as in the p4est version --- src/meshes/t8code_mesh.jl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 007da861f5b..5242ccd8ae3 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -619,8 +619,9 @@ Construct a cubed spherical shell of given inner radius and thickness as `T8code `:inside` and `:outside`. # Arguments -- `trees_per_face_dimension::Integer`: the number of trees in the first two local - dimensions of each face. +- `lat_lon_levels_per_face_dimension::Integer`: number of trees per patch in longitudinal + and latitudinal direction given as level of + refinement. - `layers::Integer`: the number of trees in the third local dimension of each face, i.e., the number of layers of the shell. - `inner_radius::Float64`: Radius of the inner side of the shell. @@ -633,11 +634,13 @@ Construct a cubed spherical shell of given inner radius and thickness as `T8code - `initial_refinement_level::Integer`: refine the mesh uniformly to this level before the simulation starts. """ -function T8codeMeshCubedSphere(trees_per_face_dimension, layers, inner_radius, thickness; +function T8codeMeshCubedSphere(lat_lon_levels_per_face_dimension, layers, inner_radius, + thickness; polydeg, RealT = Float64, initial_refinement_level = 0) NDIMS = 3 cmesh = t8_cmesh_new_cubed_spherical_shell(inner_radius, thickness, - trees_per_face_dimension, layers, mpi_comm()) + lat_lon_levels_per_face_dimension, + layers, mpi_comm()) do_face_ghost = mpi_isparallel() scheme = t8_scheme_new_default_cxx() forest = t8_forest_new_uniform(cmesh, scheme, initial_refinement_level, do_face_ghost, From c46799beb3a48bdb48af4243526d898245929cc7 Mon Sep 17 00:00:00 2001 From: Benedict Geihe Date: Mon, 22 Apr 2024 11:22:24 +0200 Subject: [PATCH 15/81] use lat lon levels --- examples/t8code_3d_dgsem/elixir_advection_cubed_sphere.jl | 3 ++- .../t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/t8code_3d_dgsem/elixir_advection_cubed_sphere.jl b/examples/t8code_3d_dgsem/elixir_advection_cubed_sphere.jl index 9163cadada8..3c0817e969d 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_cubed_sphere.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_cubed_sphere.jl @@ -17,7 +17,8 @@ boundary_condition = BoundaryConditionDirichlet(initial_condition) boundary_conditions = Dict(:inside => boundary_condition, :outside => boundary_condition) -mesh = Trixi.T8codeMeshCubedSphere(5, 3, 0.5, 0.5; +# Note that the first argument refers to the level of refinement, unlike in for p4est +mesh = Trixi.T8codeMeshCubedSphere(2, 3, 0.5, 0.5; polydeg = 3, initial_refinement_level = 0) # A semidiscretization collects data structures and functions for the spatial discretization diff --git a/examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl b/examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl index 4f5b187885e..21260288996 100644 --- a/examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl +++ b/examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl @@ -227,8 +227,11 @@ volume_flux = flux_kennedy_gruber solver = DGSEM(polydeg = 5, surface_flux = surface_flux, volume_integral = VolumeIntegralFluxDifferencing(volume_flux)) -trees_per_cube_face = (4, 4) -mesh = Trixi.T8codeMeshCubedSphere(trees_per_cube_face..., 6.371229e6, 30000.0, +# For optimal results, use 4 lat lon levels and 8 layers here +# Note that the first argument refers to the level of refinement, unlike in for p4est +lat_lon_levels = 3 +layers = 4 +mesh = Trixi.T8codeMeshCubedSphere(lat_lon_levels, layers, 6.371229e6, 30000.0, polydeg = 5, initial_refinement_level = 0) semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition, solver, From e3c2492cb181e48398f78f29ff0920b2dbed0c2c Mon Sep 17 00:00:00 2001 From: Benedict Geihe Date: Mon, 22 Apr 2024 11:22:55 +0200 Subject: [PATCH 16/81] add t8code cubed sphere tests --- test/test_t8code_3d.jl | 49 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/test/test_t8code_3d.jl b/test/test_t8code_3d.jl index 300eaef66c8..72d142fcdb7 100644 --- a/test/test_t8code_3d.jl +++ b/test/test_t8code_3d.jl @@ -99,6 +99,22 @@ mkdir(outdir) end end + # This test differs from the one in `test_p4est_3d.jl` in the latitudinal and + # longitudinal dimensions. + @trixi_testset "elixir_advection_cubed_sphere.jl" begin + @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_cubed_sphere.jl"), + l2=[0.002006918015656413], + linf=[0.027655117058380085]) + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end + end + # This test is identical to the one in `test_p4est_3d.jl`. @trixi_testset "elixir_euler_source_terms_nonconforming_unstructured_curved.jl" begin @test_trixi_include(joinpath(EXAMPLES_DIR, @@ -271,6 +287,39 @@ mkdir(outdir) @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 end end + + # This test is identical to the one in `test_p4est_3d.jl` besides minor + # deviations in the expected error norms. + @trixi_testset "elixir_euler_baroclinic_instability.jl" begin + @test_trixi_include(joinpath(EXAMPLES_DIR, + "elixir_euler_baroclinic_instability.jl"), + l2=[ + 6.725093801700048e-7, + 0.00021710076010951073, + 0.0004386796338203878, + 0.00020836270267103122, + 0.07601887903440395, + ], + linf=[ + 1.9107530539574924e-5, + 0.02980358831035801, + 0.048476331898047564, + 0.02200137344113612, + 4.848310144356219, + ], + tspan=(0.0, 1e2), + # Decrease tolerance of adaptive time stepping to get similar results across different systems + abstol=1.0e-9, reltol=1.0e-9, + coverage_override=(lat_lon_levels = 0, layers = 1, polydeg = 3)) # Prevent long compile time in CI + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end + end end # Clean up afterwards: delete Trixi.jl output directory From 7af3f31ab97e8b4fb00bdc90ea2f4713890b8dc8 Mon Sep 17 00:00:00 2001 From: Benedict <135045760+benegee@users.noreply.github.com> Date: Thu, 25 Apr 2024 17:31:55 +0200 Subject: [PATCH 17/81] Remove TODO comments --- src/meshes/t8code_mesh.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 5242ccd8ae3..ecbbf0f7975 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -647,10 +647,8 @@ function T8codeMeshCubedSphere(lat_lon_levels_per_face_dimension, layers, inner_ mpi_comm()) num_trees = t8_cmesh_get_num_trees(cmesh) - # TODO: Init?! boundary_names = fill(Symbol("---"), 2 * NDIMS, num_trees) for itree in 1:num_trees - # TODO: z-direction == radial direction in each tree? boundary_names[5, itree] = :inside boundary_names[6, itree] = :outside end From 6d835e33e0325b2d22629b6e79c1464d3fc0790d Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Fri, 26 Apr 2024 10:50:26 +0200 Subject: [PATCH 18/81] Relaxing T8code.jl version requirement. --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index a12522372de..0db43a69548 100644 --- a/Project.toml +++ b/Project.toml @@ -92,7 +92,7 @@ StaticArrays = "1.5" StrideArrays = "0.1.26" StructArrays = "0.6.11" SummationByPartsOperators = "0.5.41" -T8code = "0.6.0" +T8code = "0.4.3, 0.5, 0.6" TimerOutputs = "0.5.7" Triangulate = "2.2" TriplotBase = "0.1" From 54a5ec3d3853cc0e5371516221df2639820b6d3d Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Fri, 26 Apr 2024 11:27:09 +0200 Subject: [PATCH 19/81] Restricted t8code version requirement. --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 0db43a69548..d0ee452b2a6 100644 --- a/Project.toml +++ b/Project.toml @@ -92,7 +92,7 @@ StaticArrays = "1.5" StrideArrays = "0.1.26" StructArrays = "0.6.11" SummationByPartsOperators = "0.5.41" -T8code = "0.4.3, 0.5, 0.6" +T8code = "0.5" TimerOutputs = "0.5.7" Triangulate = "2.2" TriplotBase = "0.1" From 1daec4b7154183c85f272a729cdf8598007e536d Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Fri, 26 Apr 2024 11:28:55 +0200 Subject: [PATCH 20/81] Removed cubed spherical shell related code. --- .../elixir_advection_cubed_sphere.jl | 62 ---- .../elixir_euler_baroclinic_instability.jl | 301 ------------------ src/meshes/t8code_mesh.jl | 46 --- 3 files changed, 409 deletions(-) delete mode 100644 examples/t8code_3d_dgsem/elixir_advection_cubed_sphere.jl delete mode 100644 examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl diff --git a/examples/t8code_3d_dgsem/elixir_advection_cubed_sphere.jl b/examples/t8code_3d_dgsem/elixir_advection_cubed_sphere.jl deleted file mode 100644 index 3c0817e969d..00000000000 --- a/examples/t8code_3d_dgsem/elixir_advection_cubed_sphere.jl +++ /dev/null @@ -1,62 +0,0 @@ - -using OrdinaryDiffEq -using Trixi - -############################################################################### -# semidiscretization of the linear advection equation - -advection_velocity = (0.2, -0.7, 0.5) -equations = LinearScalarAdvectionEquation3D(advection_velocity) - -# Create DG solver with polynomial degree = 3 and (local) Lax-Friedrichs/Rusanov flux as surface flux -solver = DGSEM(polydeg = 3, surface_flux = flux_lax_friedrichs) - -initial_condition = initial_condition_convergence_test - -boundary_condition = BoundaryConditionDirichlet(initial_condition) -boundary_conditions = Dict(:inside => boundary_condition, - :outside => boundary_condition) - -# Note that the first argument refers to the level of refinement, unlike in for p4est -mesh = Trixi.T8codeMeshCubedSphere(2, 3, 0.5, 0.5; - polydeg = 3, initial_refinement_level = 0) - -# A semidiscretization collects data structures and functions for the spatial discretization -semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition, solver, - boundary_conditions = boundary_conditions) - -############################################################################### -# ODE solvers, callbacks etc. - -# Create ODE problem with time span from 0.0 to 1.0 -tspan = (0.0, 1.0) -ode = semidiscretize(semi, tspan) - -# At the beginning of the main loop, the SummaryCallback prints a summary of the simulation setup -# and resets the timers -summary_callback = SummaryCallback() - -# The AnalysisCallback allows to analyse the solution in regular intervals and prints the results -analysis_callback = AnalysisCallback(semi, interval = 100) - -# # The SaveSolutionCallback allows to save the solution to a file in regular intervals -# save_solution = SaveSolutionCallback(interval = 100, -# solution_variables = cons2prim) - -# The StepsizeCallback handles the re-calculation of the maximum Δt after each time step -stepsize_callback = StepsizeCallback(cfl = 1.2) - -# Create a CallbackSet to collect all callbacks such that they can be passed to the ODE solver -callbacks = CallbackSet(summary_callback, analysis_callback, # save_solution, - stepsize_callback) - -############################################################################### -# run the simulation - -# OrdinaryDiffEq's `solve` method evolves the solution in time and executes the passed callbacks -sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), - dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback - save_everystep = false, callback = callbacks); - -# Print the timer summary -summary_callback() diff --git a/examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl b/examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl deleted file mode 100644 index 21260288996..00000000000 --- a/examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl +++ /dev/null @@ -1,301 +0,0 @@ -# An idealized baroclinic instability test case -# For optimal results consider increasing the resolution to 16x16x8 trees per cube face. -# -# Note that this elixir can take several hours to run. -# Using 24 threads of an AMD Ryzen Threadripper 3990X (more threads don't speed it up further) -# and `check-bounds=no`, this elixirs takes about one hour to run. -# With 16x16x8 trees per cube face on the same machine, it takes about 28 hours. -# -# References: -# - Paul A. Ullrich, Thomas Melvin, Christiane Jablonowski, Andrew Staniforth (2013) -# A proposed baroclinic wave test case for deep- and shallow-atmosphere dynamical cores -# https://doi.org/10.1002/qj.2241 - -using OrdinaryDiffEq -using Trixi -using LinearAlgebra - -############################################################################### -# Setup for the baroclinic instability test -gamma = 1.4 -equations = CompressibleEulerEquations3D(gamma) - -# Initial condition for an idealized baroclinic instability test -# https://doi.org/10.1002/qj.2241, Section 3.2 and Appendix A -function initial_condition_baroclinic_instability(x, t, - equations::CompressibleEulerEquations3D) - lon, lat, r = cartesian_to_sphere(x) - radius_earth = 6.371229e6 - # Make sure that the r is not smaller than radius_earth - z = max(r - radius_earth, 0.0) - - # Unperturbed basic state - rho, u, p = basic_state_baroclinic_instability_longitudinal_velocity(lon, lat, z) - - # Stream function type perturbation - u_perturbation, v_perturbation = perturbation_stream_function(lon, lat, z) - - u += u_perturbation - v = v_perturbation - - # Convert spherical velocity to Cartesian - v1 = -sin(lon) * u - sin(lat) * cos(lon) * v - v2 = cos(lon) * u - sin(lat) * sin(lon) * v - v3 = cos(lat) * v - - return prim2cons(SVector(rho, v1, v2, v3, p), equations) -end - -# Steady state for RHS correction below -function steady_state_baroclinic_instability(x, t, equations::CompressibleEulerEquations3D) - lon, lat, r = cartesian_to_sphere(x) - radius_earth = 6.371229e6 - # Make sure that the r is not smaller than radius_earth - z = max(r - radius_earth, 0.0) - - # Unperturbed basic state - rho, u, p = basic_state_baroclinic_instability_longitudinal_velocity(lon, lat, z) - - # Convert spherical velocity to Cartesian - v1 = -sin(lon) * u - v2 = cos(lon) * u - v3 = 0.0 - - return prim2cons(SVector(rho, v1, v2, v3, p), equations) -end - -function cartesian_to_sphere(x) - r = norm(x) - lambda = atan(x[2], x[1]) - if lambda < 0 - lambda += 2 * pi - end - phi = asin(x[3] / r) - - return lambda, phi, r -end - -# Unperturbed balanced steady-state. -# Returns primitive variables with only the velocity in longitudinal direction (rho, u, p). -# The other velocity components are zero. -function basic_state_baroclinic_instability_longitudinal_velocity(lon, lat, z) - # Parameters from Table 1 in the paper - # Corresponding names in the paper are commented - radius_earth = 6.371229e6 # a - half_width_parameter = 2 # b - gravitational_acceleration = 9.80616 # g - k = 3 # k - surface_pressure = 1e5 # p₀ - gas_constant = 287 # R - surface_equatorial_temperature = 310.0 # T₀ᴱ - surface_polar_temperature = 240.0 # T₀ᴾ - lapse_rate = 0.005 # Γ - angular_velocity = 7.29212e-5 # Ω - - # Distance to the center of the Earth - r = z + radius_earth - - # In the paper: T₀ - temperature0 = 0.5 * (surface_equatorial_temperature + surface_polar_temperature) - # In the paper: A, B, C, H - const_a = 1 / lapse_rate - const_b = (temperature0 - surface_polar_temperature) / - (temperature0 * surface_polar_temperature) - const_c = 0.5 * (k + 2) * (surface_equatorial_temperature - surface_polar_temperature) / - (surface_equatorial_temperature * surface_polar_temperature) - const_h = gas_constant * temperature0 / gravitational_acceleration - - # In the paper: (r - a) / bH - scaled_z = z / (half_width_parameter * const_h) - - # Temporary variables - temp1 = exp(lapse_rate / temperature0 * z) - temp2 = exp(-scaled_z^2) - - # In the paper: ̃τ₁, ̃τ₂ - tau1 = const_a * lapse_rate / temperature0 * temp1 + - const_b * (1 - 2 * scaled_z^2) * temp2 - tau2 = const_c * (1 - 2 * scaled_z^2) * temp2 - - # In the paper: ∫τ₁(r') dr', ∫τ₂(r') dr' - inttau1 = const_a * (temp1 - 1) + const_b * z * temp2 - inttau2 = const_c * z * temp2 - - # Temporary variables - temp3 = r / radius_earth * cos(lat) - temp4 = temp3^k - k / (k + 2) * temp3^(k + 2) - - # In the paper: T - temperature = 1 / ((r / radius_earth)^2 * (tau1 - tau2 * temp4)) - - # In the paper: U, u (zonal wind, first component of spherical velocity) - big_u = gravitational_acceleration / radius_earth * k * temperature * inttau2 * - (temp3^(k - 1) - temp3^(k + 1)) - temp5 = radius_earth * cos(lat) - u = -angular_velocity * temp5 + sqrt(angular_velocity^2 * temp5^2 + temp5 * big_u) - - # Hydrostatic pressure - p = surface_pressure * - exp(-gravitational_acceleration / gas_constant * (inttau1 - inttau2 * temp4)) - - # Density (via ideal gas law) - rho = p / (gas_constant * temperature) - - return rho, u, p -end - -# Perturbation as in Equations 25 and 26 of the paper (analytical derivative) -function perturbation_stream_function(lon, lat, z) - # Parameters from Table 1 in the paper - # Corresponding names in the paper are commented - perturbation_radius = 1 / 6 # d₀ / a - perturbed_wind_amplitude = 1.0 # Vₚ - perturbation_lon = pi / 9 # Longitude of perturbation location - perturbation_lat = 2 * pi / 9 # Latitude of perturbation location - pertz = 15000 # Perturbation height cap - - # Great circle distance (d in the paper) divided by a (radius of the Earth) - # because we never actually need d without dividing by a - great_circle_distance_by_a = acos(sin(perturbation_lat) * sin(lat) + - cos(perturbation_lat) * cos(lat) * - cos(lon - perturbation_lon)) - - # In the first case, the vertical taper function is per definition zero. - # In the second case, the stream function is per definition zero. - if z > pertz || great_circle_distance_by_a > perturbation_radius - return 0.0, 0.0 - end - - # Vertical tapering of stream function - perttaper = 1.0 - 3 * z^2 / pertz^2 + 2 * z^3 / pertz^3 - - # sin/cos(pi * d / (2 * d_0)) in the paper - sin_, cos_ = sincos(0.5 * pi * great_circle_distance_by_a / perturbation_radius) - - # Common factor for both u and v - factor = 16 / (3 * sqrt(3)) * perturbed_wind_amplitude * perttaper * cos_^3 * sin_ - - u_perturbation = -factor * (-sin(perturbation_lat) * cos(lat) + - cos(perturbation_lat) * sin(lat) * cos(lon - perturbation_lon)) / - sin(great_circle_distance_by_a) - - v_perturbation = factor * cos(perturbation_lat) * sin(lon - perturbation_lon) / - sin(great_circle_distance_by_a) - - return u_perturbation, v_perturbation -end - -@inline function source_terms_baroclinic_instability(u, x, t, - equations::CompressibleEulerEquations3D) - radius_earth = 6.371229e6 # a - gravitational_acceleration = 9.80616 # g - angular_velocity = 7.29212e-5 # Ω - - r = norm(x) - # Make sure that r is not smaller than radius_earth - z = max(r - radius_earth, 0.0) - r = z + radius_earth - - du1 = zero(eltype(u)) - - # Gravity term - temp = -gravitational_acceleration * radius_earth^2 / r^3 - du2 = temp * u[1] * x[1] - du3 = temp * u[1] * x[2] - du4 = temp * u[1] * x[3] - du5 = temp * (u[2] * x[1] + u[3] * x[2] + u[4] * x[3]) - - # Coriolis term, -2Ω × ρv = -2 * angular_velocity * (0, 0, 1) × u[2:4] - du2 -= -2 * angular_velocity * u[3] - du3 -= 2 * angular_velocity * u[2] - - return SVector(du1, du2, du3, du4, du5) -end - -############################################################################### -# Start of the actual elixir, semidiscretization of the problem - -initial_condition = initial_condition_baroclinic_instability - -boundary_conditions = Dict(:inside => boundary_condition_slip_wall, - :outside => boundary_condition_slip_wall) - -# This is a good estimate for the speed of sound in this example. -# Other values between 300 and 400 should work as well. -surface_flux = FluxLMARS(340) -volume_flux = flux_kennedy_gruber -solver = DGSEM(polydeg = 5, surface_flux = surface_flux, - volume_integral = VolumeIntegralFluxDifferencing(volume_flux)) - -# For optimal results, use 4 lat lon levels and 8 layers here -# Note that the first argument refers to the level of refinement, unlike in for p4est -lat_lon_levels = 3 -layers = 4 -mesh = Trixi.T8codeMeshCubedSphere(lat_lon_levels, layers, 6.371229e6, 30000.0, - polydeg = 5, initial_refinement_level = 0) - -semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition, solver, - source_terms = source_terms_baroclinic_instability, - boundary_conditions = boundary_conditions) - -############################################################################### -# ODE solvers, callbacks etc. - -tspan = (0.0, 10 * 24 * 60 * 60.0) # time in seconds for 10 days - -# Save RHS of the steady state and subtract it in every RHS evaluation. -# This trick preserves the steady state exactly (to machine rounding errors, of course). -# Otherwise, this elixir produces entirely unusable results for a resolution of 8x8x4 cells -# per cube face with a polydeg of 3. -# With this trick, even the polydeg 3 simulation produces usable (although badly resolved) results, -# and most of the grid imprinting in higher polydeg simulation is eliminated. -# -# See https://github.com/trixi-framework/Trixi.jl/issues/980 for more information. -u_steady_state = compute_coefficients(steady_state_baroclinic_instability, tspan[1], semi) -# Use a `let` block for performance (otherwise du_steady_state will be a global variable) -let du_steady_state = similar(u_steady_state) - # Save RHS of the steady state - Trixi.rhs!(du_steady_state, u_steady_state, semi, tspan[1]) - - global function corrected_rhs!(du, u, semi, t) - # Normal RHS evaluation - Trixi.rhs!(du, u, semi, t) - # Correct by subtracting the steady-state RHS - Trixi.@trixi_timeit Trixi.timer() "rhs correction" begin - # Use Trixi.@threaded for threaded performance - Trixi.@threaded for i in eachindex(du) - du[i] -= du_steady_state[i] - end - end - end -end -u0 = compute_coefficients(tspan[1], semi) -ode = ODEProblem(corrected_rhs!, u0, tspan, semi) - -summary_callback = SummaryCallback() - -analysis_interval = 5000 -analysis_callback = AnalysisCallback(semi, interval = analysis_interval) - -alive_callback = AliveCallback(analysis_interval = analysis_interval) - -#save_solution = SaveSolutionCallback(interval = 5000, -# save_initial_solution = true, -# save_final_solution = true, -# solution_variables = cons2prim) - -callbacks = CallbackSet(summary_callback, - analysis_callback, - alive_callback) -# , save_solution) - -############################################################################### -# run the simulation - -# Use a Runge-Kutta method with automatic (error based) time step size control -# Enable threading of the RK method for better performance on multiple threads -sol = solve(ode, RDPK3SpFSAL49(thread = OrdinaryDiffEq.True()); abstol = 1.0e-6, - reltol = 1.0e-6, - ode_default_options()..., callback = callbacks); - -summary_callback() # print the timer summary diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index ecbbf0f7975..be76fb3f56b 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -610,52 +610,6 @@ function T8codeMesh(meshfile::AbaqusFile{NDIMS}; boundary_names, "") end -""" -T8codeMeshCubedSphere(trees_per_face_dimension, layers, inner_radius, thickness; - polydeg, RealT=Float64, initial_refinement_level=0) - -Construct a cubed spherical shell of given inner radius and thickness as `T8codeMesh` with -`6 * trees_per_face_dimension^2 * layers` trees. The mesh will have two boundaries, -`:inside` and `:outside`. - -# Arguments -- `lat_lon_levels_per_face_dimension::Integer`: number of trees per patch in longitudinal - and latitudinal direction given as level of - refinement. -- `layers::Integer`: the number of trees in the third local dimension of each face, i.e., - the number of layers of the shell. -- `inner_radius::Float64`: Radius of the inner side of the shell. -- `thickness::Float64`: Thickness of the shell. The outer radius will be - `inner_radius + thickness`. -- `polydeg::Integer`: polynomial degree used to store the geometry of the mesh. - The mapping will be approximated by an interpolation polynomial - of the specified degree for each tree. -- `RealT::Type`: the type that should be used for coordinates. -- `initial_refinement_level::Integer`: refine the mesh uniformly to this level before the - simulation starts. -""" -function T8codeMeshCubedSphere(lat_lon_levels_per_face_dimension, layers, inner_radius, - thickness; - polydeg, RealT = Float64, initial_refinement_level = 0) - NDIMS = 3 - cmesh = t8_cmesh_new_cubed_spherical_shell(inner_radius, thickness, - lat_lon_levels_per_face_dimension, - layers, mpi_comm()) - do_face_ghost = mpi_isparallel() - scheme = t8_scheme_new_default_cxx() - forest = t8_forest_new_uniform(cmesh, scheme, initial_refinement_level, do_face_ghost, - mpi_comm()) - - num_trees = t8_cmesh_get_num_trees(cmesh) - boundary_names = fill(Symbol("---"), 2 * NDIMS, num_trees) - for itree in 1:num_trees - boundary_names[5, itree] = :inside - boundary_names[6, itree] = :outside - end - - return T8codeMesh{NDIMS, RealT}(forest, boundary_names; polydeg = polydeg) -end - struct adapt_callback_passthrough adapt_callback::Function user_data::Any From 120af75ad8b49e284b1a32f2ee5b505686f4a70f Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Fri, 26 Apr 2024 11:33:18 +0200 Subject: [PATCH 21/81] Removed cubed spherical shell tests. --- test/test_t8code_3d.jl | 49 ------------------------------------------ 1 file changed, 49 deletions(-) diff --git a/test/test_t8code_3d.jl b/test/test_t8code_3d.jl index 72d142fcdb7..300eaef66c8 100644 --- a/test/test_t8code_3d.jl +++ b/test/test_t8code_3d.jl @@ -99,22 +99,6 @@ mkdir(outdir) end end - # This test differs from the one in `test_p4est_3d.jl` in the latitudinal and - # longitudinal dimensions. - @trixi_testset "elixir_advection_cubed_sphere.jl" begin - @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_cubed_sphere.jl"), - l2=[0.002006918015656413], - linf=[0.027655117058380085]) - # Ensure that we do not have excessive memory allocations - # (e.g., from type instabilities) - let - t = sol.t[end] - u_ode = sol.u[end] - du_ode = similar(u_ode) - @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 - end - end - # This test is identical to the one in `test_p4est_3d.jl`. @trixi_testset "elixir_euler_source_terms_nonconforming_unstructured_curved.jl" begin @test_trixi_include(joinpath(EXAMPLES_DIR, @@ -287,39 +271,6 @@ mkdir(outdir) @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 end end - - # This test is identical to the one in `test_p4est_3d.jl` besides minor - # deviations in the expected error norms. - @trixi_testset "elixir_euler_baroclinic_instability.jl" begin - @test_trixi_include(joinpath(EXAMPLES_DIR, - "elixir_euler_baroclinic_instability.jl"), - l2=[ - 6.725093801700048e-7, - 0.00021710076010951073, - 0.0004386796338203878, - 0.00020836270267103122, - 0.07601887903440395, - ], - linf=[ - 1.9107530539574924e-5, - 0.02980358831035801, - 0.048476331898047564, - 0.02200137344113612, - 4.848310144356219, - ], - tspan=(0.0, 1e2), - # Decrease tolerance of adaptive time stepping to get similar results across different systems - abstol=1.0e-9, reltol=1.0e-9, - coverage_override=(lat_lon_levels = 0, layers = 1, polydeg = 3)) # Prevent long compile time in CI - # Ensure that we do not have excessive memory allocations - # (e.g., from type instabilities) - let - t = sol.t[end] - u_ode = sol.u[end] - du_ode = similar(u_ode) - @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 - end - end end # Clean up afterwards: delete Trixi.jl output directory From 4317d5ee7e3068a935bce2dfed75f4eef47e67af Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Fri, 26 Apr 2024 14:50:11 +0200 Subject: [PATCH 22/81] Increasing code coverage. --- test/test_t8code_2d.jl | 17 +++++++++++++++++ test/test_t8code_3d.jl | 11 +++++++++++ 2 files changed, 28 insertions(+) diff --git a/test/test_t8code_2d.jl b/test/test_t8code_2d.jl index d536a6dd73a..d99775f743d 100644 --- a/test/test_t8code_2d.jl +++ b/test/test_t8code_2d.jl @@ -30,6 +30,12 @@ mkdir(outdir) end end +@trixi_testset "test load mesh from path" begin + @test_throws "Unknown file extension: .unknown_ext" begin + mesh = T8codeMesh(touch("dummy.unknown_ext"), 2) + end +end + @trixi_testset "test check_for_negative_volumes" begin @test_warn "Discovered negative volumes" begin # Unstructured mesh with six cells which have left-handed node ordering. @@ -42,6 +48,17 @@ end end end +@trixi_testset "test t8code mesh from p4est connectivity" begin + @test begin + # Here we use the connectivity constructor from `P4est.jl` since the + # method dispatch works only on `Ptr{p4est_connectivity}` which + # actually is `Ptr{P4est.LibP4est.p4est_connectivity}`. + conn = Trixi.P4est.LibP4est.p4est_connectivity_new_brick(2, 3, 1, 1) + mesh = T8codeMesh(conn) + all(size(mesh.tree_node_coordinates) .== (2, 2, 2, 6)) + end +end + @trixi_testset "elixir_advection_basic.jl" begin # This test is identical to the one in `test_p4est_2d.jl`. @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_basic.jl"), diff --git a/test/test_t8code_3d.jl b/test/test_t8code_3d.jl index 300eaef66c8..81d2a7cdd85 100644 --- a/test/test_t8code_3d.jl +++ b/test/test_t8code_3d.jl @@ -13,6 +13,17 @@ isdir(outdir) && rm(outdir, recursive = true) mkdir(outdir) @testset "T8codeMesh3D" begin + @trixi_testset "test t8code mesh from p8est connectivity" begin + @test begin + # Here we use the connectivity constructor from `P4est.jl` since the + # method dispatch works only on `Ptr{p8est_connectivity}` which + # actually is `Ptr{P4est.LibP4est.p8est_connectivity}`. + conn = Trixi.P4est.LibP4est.p8est_connectivity_new_brick(2, 3, 4, 1, 1, 1) + mesh = T8codeMesh(conn) + all(size(mesh.tree_node_coordinates) .== (3, 2, 2, 2, 24)) + end + end + # This test is identical to the one in `test_p4est_3d.jl`. @trixi_testset "elixir_advection_basic.jl" begin @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_basic.jl"), From 4b5694dc96e0dc2d1702bc4333d42ad61123866b Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Tue, 30 Apr 2024 17:28:59 +0200 Subject: [PATCH 23/81] Increasing code coverage. --- .../elixir_advection_nonconforming_flag.jl | 3 ++- src/meshes/t8code_mesh.jl | 17 ++++++++++------- test/test_t8code_2d.jl | 10 ++++++++++ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl b/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl index a39f3a7e195..4161f85e380 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl @@ -25,7 +25,8 @@ mapping = Trixi.transfinite_mapping(faces) trees_per_dimension = (3, 2) mesh = T8codeMesh(trees_per_dimension, polydeg = 3, mapping = mapping, - initial_refinement_level = 1) + initial_refinement_level = 1, + periodicity = (true, true)) # Note: This is actually a `p4est_quadrant_t` which is much bigger than the # following struct. But we only need the first three fields for our purpose. diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index be76fb3f56b..01033b0d6a1 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -593,13 +593,8 @@ function T8codeMesh(meshfile::AbaqusFile{NDIMS}; boundary_symbols) end - if typeof(connectivity) <: Ptr{p4est_connectivity} - cmesh = t8_cmesh_new_from_p4est(connectivity, mpi_comm(), 0) - elseif typeof(connectivity) <: Ptr{p8est_connectivity} - cmesh = t8_cmesh_new_from_p8est(connectivity, mpi_comm(), 0) - else - throw("`connectivity` is not of type `Ptr{p*est_connectivity}`.") - end + cmesh = t8_cmesh_new_from_connectivity(connectivity, mpi_comm()) + p4est_connectivity_destroy(connectivity) do_face_ghost = mpi_isparallel() scheme = t8_scheme_new_default_cxx() @@ -610,6 +605,14 @@ function T8codeMesh(meshfile::AbaqusFile{NDIMS}; boundary_names, "") end +function t8_cmesh_new_from_connectivity(connectivity::Ptr{p4est_connectivity}, comm) + return t8_cmesh_new_from_p4est(connectivity, comm, 0) +end + +function t8_cmesh_new_from_connectivity(connectivity::Ptr{p8est_connectivity}, comm) + return t8_cmesh_new_from_p8est(connectivity, comm, 0) +end + struct adapt_callback_passthrough adapt_callback::Function user_data::Any diff --git a/test/test_t8code_2d.jl b/test/test_t8code_2d.jl index d99775f743d..7f318b9fd2d 100644 --- a/test/test_t8code_2d.jl +++ b/test/test_t8code_2d.jl @@ -59,6 +59,16 @@ end end end +@trixi_testset "test t8code mesh from ABAQUS HOHQMesh file" begin + @test begin + # Unstructured ABAQUS mesh file created with HOHQMesh.. + file_path = Trixi.download("https://gist.githubusercontent.com/jmark/9e0da4306e266617eeb19bc56b0e7feb/raw/e6856e1deb648a807f6bb6d6dcacff9e55d94e2a/round_2d_tank.inp", + joinpath(EXAMPLES_DIR, "round_2d_tank.inp")) + mesh = T8codeMesh(AbaqusFile{2}(file_path)) + all(size(mesh.tree_node_coordinates) .== (2, 4, 4, 340)) + end +end + @trixi_testset "elixir_advection_basic.jl" begin # This test is identical to the one in `test_p4est_2d.jl`. @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_basic.jl"), From 48082e6cfa3ddb26dc5da761bebde9f62d8fc77e Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Tue, 30 Apr 2024 17:34:57 +0200 Subject: [PATCH 24/81] Further increasing code coverage. --- .../t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl b/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl index 4161f85e380..48f78dd6da3 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl @@ -18,13 +18,12 @@ f3(s) = SVector(s, -1.0 + sin(0.5 * pi * s)) f4(s) = SVector(s, 1.0 + sin(0.5 * pi * s)) faces = (f1, f2, f3, f4) -mapping = Trixi.transfinite_mapping(faces) # Create T8codeMesh with 3 x 2 trees and 6 x 4 elements, # approximate the geometry with a smaller polydeg for testing. trees_per_dimension = (3, 2) mesh = T8codeMesh(trees_per_dimension, polydeg = 3, - mapping = mapping, + faces = faces, initial_refinement_level = 1, periodicity = (true, true)) From 72c5c929984020fd8b9ffae1aeebccd71554abf1 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Tue, 30 Apr 2024 17:49:01 +0200 Subject: [PATCH 25/81] Applied formatter. --- test/test_t8code_2d.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_t8code_2d.jl b/test/test_t8code_2d.jl index 7f318b9fd2d..01917004caa 100644 --- a/test/test_t8code_2d.jl +++ b/test/test_t8code_2d.jl @@ -63,7 +63,7 @@ end @test begin # Unstructured ABAQUS mesh file created with HOHQMesh.. file_path = Trixi.download("https://gist.githubusercontent.com/jmark/9e0da4306e266617eeb19bc56b0e7feb/raw/e6856e1deb648a807f6bb6d6dcacff9e55d94e2a/round_2d_tank.inp", - joinpath(EXAMPLES_DIR, "round_2d_tank.inp")) + joinpath(EXAMPLES_DIR, "round_2d_tank.inp")) mesh = T8codeMesh(AbaqusFile{2}(file_path)) all(size(mesh.tree_node_coordinates) .== (2, 4, 4, 340)) end From 84e549aa164a6d316011ae1b3f9a3a6280fde4c7 Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Fri, 3 May 2024 10:36:49 +0200 Subject: [PATCH 26/81] Update src/meshes/t8code_mesh.jl Co-authored-by: Joshua Lampert <51029046+JoshuaLampert@users.noreply.github.com> --- src/meshes/t8code_mesh.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 01033b0d6a1..5365aa5fd27 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -106,9 +106,9 @@ end Create a 'T8codeMesh'. # Arguments -- 'forest': Pointer to a committed forest. -- 'boundary_names': List of boundary names. -- 'polydeg::Integer': Polynomial degree used to store the geometry of the mesh. +- `forest`: Pointer to a committed forest. +- `boundary_names`: List of boundary names. +- `polydeg::Integer`: Polynomial degree used to store the geometry of the mesh. The mapping will be approximated by an interpolation polynomial of the specified degree for each tree. - `mapping`: a function of `NDIMS` variables to describe the mapping that transforms From e08c0f8c3a64190d54467edcad6379fb7e93f6ac Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Fri, 3 May 2024 10:36:59 +0200 Subject: [PATCH 27/81] Update src/meshes/t8code_mesh.jl Co-authored-by: Joshua Lampert <51029046+JoshuaLampert@users.noreply.github.com> --- src/meshes/t8code_mesh.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 5365aa5fd27..ce2031342b5 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -127,7 +127,7 @@ function T8codeMesh{NDIMS, RealT}(forest::Ptr{t8_forest}, boundary_names; polyde ntuple(_ -> length(nodes), NDIMS)..., num_trees) - coords_ref = Vector{Cdouble}(undef, 3) + coords_ref = Vector{Float64}(undef, 3) # Calculate node coordinates of reference mesh. if NDIMS == 2 From 671702d953d88458c6d85aa99428979a9d2e7807 Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Fri, 3 May 2024 10:37:19 +0200 Subject: [PATCH 28/81] Update src/meshes/t8code_mesh.jl Co-authored-by: Benedict <135045760+benegee@users.noreply.github.com> --- src/meshes/t8code_mesh.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index ce2031342b5..bcfbf2bed22 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -508,7 +508,7 @@ end Main mesh constructor for the `T8codeMesh` that imports an unstructured, conforming mesh from an Abaqus mesh file (`.inp`). -To create a curved unstructured mesh `T8codeMesh` two strategies are available: +To create a curved unstructured `T8codeMesh` two strategies are available: - `HOHQMesh Abaqus`: High-order, curved boundary information created by [`HOHQMesh.jl`](https://github.com/trixi-framework/HOHQMesh.jl) is From aee2086ca33b1a0b0f67758cceac244e9197757f Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Fri, 3 May 2024 10:37:38 +0200 Subject: [PATCH 29/81] Update src/meshes/t8code_mesh.jl Co-authored-by: Benedict <135045760+benegee@users.noreply.github.com> --- src/meshes/t8code_mesh.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index bcfbf2bed22..85831070a9c 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -520,7 +520,7 @@ To create a curved unstructured `T8codeMesh` two strategies are available: conditions can be set at each named boundary on a given tree. - `Standard Abaqus`: By default, with `mapping=nothing` and `polydeg=1`, this creates a - straight-sided from the information parsed from the `meshfile`. If a mapping + straight-sided mesh from the information parsed from the `meshfile`. If a mapping function is specified then it computes the mapped tree coordinates via polynomial interpolants with degree `polydeg`. The mesh created by this function will only have one boundary `:all` if `boundary_symbols` is not specified. From 17288938a7ee3b39950be740e9afb560738f833a Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Fri, 3 May 2024 10:37:50 +0200 Subject: [PATCH 30/81] Update src/meshes/t8code_mesh.jl Co-authored-by: Benedict <135045760+benegee@users.noreply.github.com> --- src/meshes/t8code_mesh.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 85831070a9c..c3cd964974d 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -545,7 +545,7 @@ For example, if a two-dimensional base mesh contains 25 elements then setting `initial_refinement_level=1` creates an initial forest of `2^2 * 25 = 100` trees. # Arguments -- `meshfile::AbaqusFile{NDIMS}`: Abaqus mesh file object of dimension NDIMS and give `path` to the file. +- `meshfile::AbaqusFile{NDIMS}`: Abaqus mesh file object of dimension NDIMS and given `path` to the file. # Optional Keyword Arguments - `mapping`: A function of `NDIMS` variables to describe the mapping that transforms From b075f5c2238a3890b19aca68a0ef057f0a338d6e Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Fri, 3 May 2024 10:38:08 +0200 Subject: [PATCH 31/81] Update src/meshes/t8code_mesh.jl Co-authored-by: Benedict <135045760+benegee@users.noreply.github.com> --- src/meshes/t8code_mesh.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index c3cd964974d..061dccca443 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -569,7 +569,7 @@ function T8codeMesh(meshfile::AbaqusFile{NDIMS}; # Read in the Header of the meshfile to determine which constructor is appropriate. header = open(meshfile.path, "r") do io - readline(io) # *Header of the Abaqus file; discarded + readline(io) # Header of the Abaqus file; discarded readline(io) # Readin the actual header information end From f2f5995f0561f38f6c9448e6ecdbdfb376fd655a Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Fri, 3 May 2024 10:38:20 +0200 Subject: [PATCH 32/81] Update src/meshes/t8code_mesh.jl Co-authored-by: Benedict <135045760+benegee@users.noreply.github.com> --- src/meshes/t8code_mesh.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 061dccca443..a1c332d894a 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -570,7 +570,7 @@ function T8codeMesh(meshfile::AbaqusFile{NDIMS}; # Read in the Header of the meshfile to determine which constructor is appropriate. header = open(meshfile.path, "r") do io readline(io) # Header of the Abaqus file; discarded - readline(io) # Readin the actual header information + readline(io) # Read in the actual header information end # Check if the meshfile was generated using HOHQMesh. From d7d85991939a5b88c6e0f85f1578fec552bba325 Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Fri, 3 May 2024 11:57:54 +0200 Subject: [PATCH 33/81] Update src/meshes/t8code_mesh.jl Co-authored-by: Joshua Lampert <51029046+JoshuaLampert@users.noreply.github.com> --- src/meshes/t8code_mesh.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index a1c332d894a..75eded53b49 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -101,7 +101,7 @@ function Base.show(io::IO, ::MIME"text/plain", mesh::T8codeMesh) end """ - T8codeMesh(forest, boundary_names; polydeg, mapping=identity) + T8codeMesh{NDIMS, RealT}(forest, boundary_names; polydeg = 1, mapping=identity) Create a 'T8codeMesh'. From 896711cda8008502165c7ad46b8c6bdb5b4e0053 Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Wed, 8 May 2024 14:52:55 +0200 Subject: [PATCH 34/81] Update test/test_t8code_2d.jl Co-authored-by: Michael Schlottke-Lakemper --- test/test_t8code_2d.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_t8code_2d.jl b/test/test_t8code_2d.jl index 01917004caa..29d84153546 100644 --- a/test/test_t8code_2d.jl +++ b/test/test_t8code_2d.jl @@ -31,8 +31,10 @@ mkdir(outdir) end @trixi_testset "test load mesh from path" begin - @test_throws "Unknown file extension: .unknown_ext" begin - mesh = T8codeMesh(touch("dummy.unknown_ext"), 2) + mktempdir() do path + @test_throws "Unknown file extension: .unknown_ext" begin + mesh = T8codeMesh(touch(joinpath(path, "dummy.unknown_ext")), 2) + end end end From af49b37a4e887985bfa6f7dbd5b9ff16b9ce44bf Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Wed, 8 May 2024 14:53:22 +0200 Subject: [PATCH 35/81] Update src/meshes/t8code_mesh.jl Co-authored-by: Hendrik Ranocha --- src/meshes/t8code_mesh.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 75eded53b49..39d2c4d76b9 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -209,6 +209,8 @@ function T8codeMesh{NDIMS, RealT}(forest::Ptr{t8_forest}, boundary_names; polyde t8_geometry_evaluate(cmesh, itree - 1, coords_ref, 1, @view(tree_node_coordinates[:, i, j, k, itree])) end + else + throw(ArgumentError("$NDIMS dimensions are not supported.")) end end From 2958b46b440e144506a397f028088d8dfd986931 Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Wed, 8 May 2024 14:54:08 +0200 Subject: [PATCH 36/81] Update src/meshes/t8code_mesh.jl Co-authored-by: Hendrik Ranocha --- src/meshes/t8code_mesh.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 39d2c4d76b9..c624aabba17 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -467,7 +467,7 @@ function T8codeMesh(filepath::String, NDIMS; kwargs...) return T8codeMesh(AbaqusFile{NDIMS}(filepath); kwargs...) end - throw("Unknown file extension: " * file_extension) + throw(ArgumentError("Unknown file extension: " * file_extension)) end """ From 324089707296c13726ec2f8ba3cd77d99ad3241e Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Wed, 8 May 2024 15:05:06 +0200 Subject: [PATCH 37/81] Update src/meshes/t8code_mesh.jl Co-authored-by: Michael Schlottke-Lakemper --- src/meshes/t8code_mesh.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index c624aabba17..bca0e62d9d9 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -156,7 +156,7 @@ function T8codeMesh{NDIMS, RealT}(forest::Ptr{t8_forest}, boundary_names; polyde vol = dot(cross(u, v), w) if vol < z - @warn "Discovered negative volumes in `cmesh`: vol = $vol" + error("Discovered negative volumes in `cmesh`: vol = $vol") end end From 9c57b1330d071e1704ff98027778517daad1738a Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Wed, 8 May 2024 15:06:08 +0200 Subject: [PATCH 38/81] Fixing minor stuff. --- src/Trixi.jl | 2 - src/meshes/t8code_mesh.jl | 8 +- src/solvers/dgsem_t8code/containers_2d.jl | 1 - src/solvers/dgsem_t8code/containers_3d.jl | 1 - test/test_t8code_2d.jl | 514 +++++++++++----------- 5 files changed, 262 insertions(+), 264 deletions(-) diff --git a/src/Trixi.jl b/src/Trixi.jl index 8e5460f6449..5511f7e02e2 100644 --- a/src/Trixi.jl +++ b/src/Trixi.jl @@ -289,8 +289,6 @@ export PlotData1D, PlotData2D, ScalarPlotData2D, getmesh, adapt_to_mesh_level!, adapt_to_mesh_level, iplot, iplot! -export GmshFile, AbaqusFile - function __init__() init_mpi() diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 75eded53b49..3a157ade4d3 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -436,7 +436,7 @@ by the file extension. # Arguments - `filepath::String`: path to a Gmsh or Abaqus mesh file. -- `NDIMS`: Mesh file dimension: `2` or `3`. +- `ndims`: Mesh file dimension: `2` or `3`. # Optional Keyword Arguments - `mapping`: A function of `NDIMS` variables to describe the mapping that transforms @@ -449,7 +449,7 @@ by the file extension. - `RealT::Type`: The type that should be used for coordinates. - `initial_refinement_level::Integer`: Refine the mesh uniformly to this level before the simulation starts. """ -function T8codeMesh(filepath::String, NDIMS; kwargs...) +function T8codeMesh(filepath::String, ndims; kwargs...) # Prevent `t8code` from crashing Julia if the file doesn't exist. @assert isfile(filepath) @@ -458,11 +458,11 @@ function T8codeMesh(filepath::String, NDIMS; kwargs...) file_extension = lowercase(meshfile_suffix) if file_extension == ".msh" - return T8codeMesh(GmshFile{NDIMS}(filepath); kwargs...) + return T8codeMesh(GmshFile{ndims}(filepath); kwargs...) end if file_extension == ".inp" - return T8codeMesh(AbaqusFile{NDIMS}(filepath); kwargs...) + return T8codeMesh(AbaqusFile{ndims}(filepath); kwargs...) end throw("Unknown file extension: " * file_extension) diff --git a/src/solvers/dgsem_t8code/containers_2d.jl b/src/solvers/dgsem_t8code/containers_2d.jl index 104e5590a58..fb17b45d21e 100644 --- a/src/solvers/dgsem_t8code/containers_2d.jl +++ b/src/solvers/dgsem_t8code/containers_2d.jl @@ -3,7 +3,6 @@ # we need to opt-in explicitly. # See https://ranocha.de/blog/Optimizing_EC_Trixi for further details. @muladd begin -#! format: noindent #! format: off # Interpolate tree_node_coordinates to each quadrant at the specified nodes. diff --git a/src/solvers/dgsem_t8code/containers_3d.jl b/src/solvers/dgsem_t8code/containers_3d.jl index e1e58fafd85..364877a09cf 100644 --- a/src/solvers/dgsem_t8code/containers_3d.jl +++ b/src/solvers/dgsem_t8code/containers_3d.jl @@ -3,7 +3,6 @@ # we need to opt-in explicitly. # See https://ranocha.de/blog/Optimizing_EC_Trixi for further details. @muladd begin -#! format: noindent #! format: off # Interpolate tree_node_coordinates to each quadrant at the specified nodes diff --git a/test/test_t8code_2d.jl b/test/test_t8code_2d.jl index 01917004caa..5f307acd632 100644 --- a/test/test_t8code_2d.jl +++ b/test/test_t8code_2d.jl @@ -31,8 +31,10 @@ mkdir(outdir) end @trixi_testset "test load mesh from path" begin - @test_throws "Unknown file extension: .unknown_ext" begin - mesh = T8codeMesh(touch("dummy.unknown_ext"), 2) + mktempdir() do path + @test_throws "Unknown file extension: .unknown_ext" begin + mesh = T8codeMesh(touch("dummy.unknown_ext"), 2) + end end end @@ -69,260 +71,260 @@ end end end -@trixi_testset "elixir_advection_basic.jl" begin - # This test is identical to the one in `test_p4est_2d.jl`. - @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_basic.jl"), - # Expected errors are exactly the same as with TreeMesh! - l2=[8.311947673061856e-6], - linf=[6.627000273229378e-5]) - # Ensure that we do not have excessive memory allocations - # (e.g., from type instabilities) - let - t = sol.t[end] - u_ode = sol.u[end] - du_ode = similar(u_ode) - @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 - end -end - -@trixi_testset "elixir_advection_nonconforming_flag.jl" begin - # This test is identical to the one in `test_p4est_2d.jl`. - @test_trixi_include(joinpath(EXAMPLES_DIR, - "elixir_advection_nonconforming_flag.jl"), - l2=[3.198940059144588e-5], - linf=[0.00030636069494005547]) - # Ensure that we do not have excessive memory allocations - # (e.g., from type instabilities) - let - t = sol.t[end] - u_ode = sol.u[end] - du_ode = similar(u_ode) - @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 - end -end - -@trixi_testset "elixir_advection_unstructured_flag.jl" begin - # This test is identical to the one in `test_p4est_2d.jl`. - @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_unstructured_flag.jl"), - l2=[0.0005379687442422346], - linf=[0.007438525029884735]) - # Ensure that we do not have excessive memory allocations - # (e.g., from type instabilities) - let - t = sol.t[end] - u_ode = sol.u[end] - du_ode = similar(u_ode) - @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 - end -end - -@trixi_testset "elixir_advection_amr_unstructured_flag.jl" begin - @test_trixi_include(joinpath(EXAMPLES_DIR, - "elixir_advection_amr_unstructured_flag.jl"), - l2=[0.001993165013217687], - linf=[0.032891018571625796], - coverage_override=(maxiters = 6,)) - # Ensure that we do not have excessive memory allocations - # (e.g., from type instabilities) - let - t = sol.t[end] - u_ode = sol.u[end] - du_ode = similar(u_ode) - @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 - end -end - -@trixi_testset "elixir_advection_amr_solution_independent.jl" begin - # This test is identical to the one in `test_p4est_2d.jl`. - @test_trixi_include(joinpath(EXAMPLES_DIR, - "elixir_advection_amr_solution_independent.jl"), - # Expected errors are exactly the same as with StructuredMesh! - l2=[4.949660644033807e-5], - linf=[0.0004867846262313763], - coverage_override=(maxiters = 6,)) - # Ensure that we do not have excessive memory allocations - # (e.g., from type instabilities) - let - t = sol.t[end] - u_ode = sol.u[end] - du_ode = similar(u_ode) - @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 - end -end - -@trixi_testset "elixir_euler_source_terms_nonconforming_unstructured_flag.jl" begin - # This test is identical to the one in `test_p4est_2d.jl`. - @test_trixi_include(joinpath(EXAMPLES_DIR, - "elixir_euler_source_terms_nonconforming_unstructured_flag.jl"), - l2=[ - 0.0034516244508588046, - 0.0023420334036925493, - 0.0024261923964557187, - 0.004731710454271893, - ], - linf=[ - 0.04155789011775046, - 0.024772109862748914, - 0.03759938693042297, - 0.08039824959535657, - ]) - # Ensure that we do not have excessive memory allocations - # (e.g., from type instabilities) - let - t = sol.t[end] - u_ode = sol.u[end] - du_ode = similar(u_ode) - @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 - end -end - -@trixi_testset "elixir_euler_free_stream.jl" begin - # This test is identical to the one in `test_p4est_2d.jl`. - @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_euler_free_stream.jl"), - l2=[ - 2.063350241405049e-15, - 1.8571016296925367e-14, - 3.1769447886391905e-14, - 1.4104095258528071e-14, - ], - linf=[1.9539925233402755e-14, 2e-12, 4.8e-12, 4e-12], - atol=2.0e-12,) - # Ensure that we do not have excessive memory allocations - # (e.g., from type instabilities) - let - t = sol.t[end] - u_ode = sol.u[end] - du_ode = similar(u_ode) - @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 - end -end - -@trixi_testset "elixir_euler_shockcapturing_ec.jl" begin - # This test is identical to the one in `test_p4est_2d.jl`. - @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_euler_shockcapturing_ec.jl"), - l2=[ - 9.53984675e-02, - 1.05633455e-01, - 1.05636158e-01, - 3.50747237e-01, - ], - linf=[ - 2.94357464e-01, - 4.07893014e-01, - 3.97334516e-01, - 1.08142520e+00, - ], - tspan=(0.0, 1.0)) - # Ensure that we do not have excessive memory allocations - # (e.g., from type instabilities) - let - t = sol.t[end] - u_ode = sol.u[end] - du_ode = similar(u_ode) - @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 - end -end - -@trixi_testset "elixir_euler_sedov.jl" begin - # This test is identical to the one in `test_p4est_2d.jl` besides minor - # deviations in the expected error norms. - @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_euler_sedov.jl"), - l2=[ - 3.76149952e-01, - 2.46970327e-01, - 2.46970327e-01, - 1.28889042e+00, - ], - linf=[ - 1.22139001e+00, - 1.17742626e+00, - 1.17742626e+00, - 6.20638482e+00, - ], - tspan=(0.0, 0.3)) - # Ensure that we do not have excessive memory allocations - # (e.g., from type instabilities) - let - t = sol.t[end] - u_ode = sol.u[end] - du_ode = similar(u_ode) - @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 - end -end - -@trixi_testset "elixir_shallowwater_source_terms.jl" begin - # This test is identical to the one in `test_p4est_2d.jl`. - @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_shallowwater_source_terms.jl"), - l2=[ - 9.168126407325352e-5, - 0.0009795410115453788, - 0.002546408320320785, - 3.941189812642317e-6, - ], - linf=[ - 0.0009903782521019089, - 0.0059752684687262025, - 0.010941106525454103, - 1.2129488214718265e-5, - ], - tspan=(0.0, 0.1)) - # Ensure that we do not have excessive memory allocations - # (e.g., from type instabilities) - let - t = sol.t[end] - u_ode = sol.u[end] - du_ode = similar(u_ode) - @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 - end -end - -@trixi_testset "elixir_mhd_alfven_wave.jl" begin - # This test is identical to the one in `test_p4est_2d.jl`. - @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_mhd_alfven_wave.jl"), - l2=[1.0513414461545583e-5, 1.0517900957166411e-6, - 1.0517900957304043e-6, 1.511816606372376e-6, - 1.0443997728645063e-6, 7.879639064990798e-7, - 7.879639065049896e-7, 1.0628631669056271e-6, - 4.3382328912336153e-7], - linf=[4.255466285174592e-5, 1.0029706745823264e-5, - 1.0029706747467781e-5, 1.2122265939010224e-5, - 5.4791097160444835e-6, 5.18922042269665e-6, - 5.189220422141538e-6, 9.552667261422676e-6, - 1.4237578427628152e-6]) - # Ensure that we do not have excessive memory allocations - # (e.g., from type instabilities) - let - t = sol.t[end] - u_ode = sol.u[end] - du_ode = similar(u_ode) - @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 - end -end - -@trixi_testset "elixir_mhd_rotor.jl" begin - # This test is identical to the one in `test_p4est_2d.jl` besides minor - # deviations in the expected error norms. - @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_mhd_rotor.jl"), - l2=[0.44211360369891683, 0.8805178316216257, 0.8262710688468049, - 0.0, - 0.9616090460973586, 0.10386643568745411, - 0.15403457366543802, 0.0, - 2.8399715649715473e-5], - linf=[10.04369305341599, 17.995640564998403, 9.576041548174265, - 0.0, - 19.429658884314534, 1.3821395681242314, 1.818559351543182, - 0.0, - 0.002261930217575465], - tspan=(0.0, 0.02)) - # Ensure that we do not have excessive memory allocations - # (e.g., from type instabilities) - let - t = sol.t[end] - u_ode = sol.u[end] - du_ode = similar(u_ode) - @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 - end -end +## @trixi_testset "elixir_advection_basic.jl" begin +## # This test is identical to the one in `test_p4est_2d.jl`. +## @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_basic.jl"), +## # Expected errors are exactly the same as with TreeMesh! +## l2=[8.311947673061856e-6], +## linf=[6.627000273229378e-5]) +## # Ensure that we do not have excessive memory allocations +## # (e.g., from type instabilities) +## let +## t = sol.t[end] +## u_ode = sol.u[end] +## du_ode = similar(u_ode) +## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 +## end +## end +## +## @trixi_testset "elixir_advection_nonconforming_flag.jl" begin +## # This test is identical to the one in `test_p4est_2d.jl`. +## @test_trixi_include(joinpath(EXAMPLES_DIR, +## "elixir_advection_nonconforming_flag.jl"), +## l2=[3.198940059144588e-5], +## linf=[0.00030636069494005547]) +## # Ensure that we do not have excessive memory allocations +## # (e.g., from type instabilities) +## let +## t = sol.t[end] +## u_ode = sol.u[end] +## du_ode = similar(u_ode) +## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 +## end +## end +## +## @trixi_testset "elixir_advection_unstructured_flag.jl" begin +## # This test is identical to the one in `test_p4est_2d.jl`. +## @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_unstructured_flag.jl"), +## l2=[0.0005379687442422346], +## linf=[0.007438525029884735]) +## # Ensure that we do not have excessive memory allocations +## # (e.g., from type instabilities) +## let +## t = sol.t[end] +## u_ode = sol.u[end] +## du_ode = similar(u_ode) +## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 +## end +## end +## +## @trixi_testset "elixir_advection_amr_unstructured_flag.jl" begin +## @test_trixi_include(joinpath(EXAMPLES_DIR, +## "elixir_advection_amr_unstructured_flag.jl"), +## l2=[0.001993165013217687], +## linf=[0.032891018571625796], +## coverage_override=(maxiters = 6,)) +## # Ensure that we do not have excessive memory allocations +## # (e.g., from type instabilities) +## let +## t = sol.t[end] +## u_ode = sol.u[end] +## du_ode = similar(u_ode) +## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 +## end +## end +## +## @trixi_testset "elixir_advection_amr_solution_independent.jl" begin +## # This test is identical to the one in `test_p4est_2d.jl`. +## @test_trixi_include(joinpath(EXAMPLES_DIR, +## "elixir_advection_amr_solution_independent.jl"), +## # Expected errors are exactly the same as with StructuredMesh! +## l2=[4.949660644033807e-5], +## linf=[0.0004867846262313763], +## coverage_override=(maxiters = 6,)) +## # Ensure that we do not have excessive memory allocations +## # (e.g., from type instabilities) +## let +## t = sol.t[end] +## u_ode = sol.u[end] +## du_ode = similar(u_ode) +## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 +## end +## end +## +## @trixi_testset "elixir_euler_source_terms_nonconforming_unstructured_flag.jl" begin +## # This test is identical to the one in `test_p4est_2d.jl`. +## @test_trixi_include(joinpath(EXAMPLES_DIR, +## "elixir_euler_source_terms_nonconforming_unstructured_flag.jl"), +## l2=[ +## 0.0034516244508588046, +## 0.0023420334036925493, +## 0.0024261923964557187, +## 0.004731710454271893, +## ], +## linf=[ +## 0.04155789011775046, +## 0.024772109862748914, +## 0.03759938693042297, +## 0.08039824959535657, +## ]) +## # Ensure that we do not have excessive memory allocations +## # (e.g., from type instabilities) +## let +## t = sol.t[end] +## u_ode = sol.u[end] +## du_ode = similar(u_ode) +## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 +## end +## end +## +## @trixi_testset "elixir_euler_free_stream.jl" begin +## # This test is identical to the one in `test_p4est_2d.jl`. +## @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_euler_free_stream.jl"), +## l2=[ +## 2.063350241405049e-15, +## 1.8571016296925367e-14, +## 3.1769447886391905e-14, +## 1.4104095258528071e-14, +## ], +## linf=[1.9539925233402755e-14, 2e-12, 4.8e-12, 4e-12], +## atol=2.0e-12,) +## # Ensure that we do not have excessive memory allocations +## # (e.g., from type instabilities) +## let +## t = sol.t[end] +## u_ode = sol.u[end] +## du_ode = similar(u_ode) +## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 +## end +## end +## +## @trixi_testset "elixir_euler_shockcapturing_ec.jl" begin +## # This test is identical to the one in `test_p4est_2d.jl`. +## @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_euler_shockcapturing_ec.jl"), +## l2=[ +## 9.53984675e-02, +## 1.05633455e-01, +## 1.05636158e-01, +## 3.50747237e-01, +## ], +## linf=[ +## 2.94357464e-01, +## 4.07893014e-01, +## 3.97334516e-01, +## 1.08142520e+00, +## ], +## tspan=(0.0, 1.0)) +## # Ensure that we do not have excessive memory allocations +## # (e.g., from type instabilities) +## let +## t = sol.t[end] +## u_ode = sol.u[end] +## du_ode = similar(u_ode) +## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 +## end +## end +## +## @trixi_testset "elixir_euler_sedov.jl" begin +## # This test is identical to the one in `test_p4est_2d.jl` besides minor +## # deviations in the expected error norms. +## @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_euler_sedov.jl"), +## l2=[ +## 3.76149952e-01, +## 2.46970327e-01, +## 2.46970327e-01, +## 1.28889042e+00, +## ], +## linf=[ +## 1.22139001e+00, +## 1.17742626e+00, +## 1.17742626e+00, +## 6.20638482e+00, +## ], +## tspan=(0.0, 0.3)) +## # Ensure that we do not have excessive memory allocations +## # (e.g., from type instabilities) +## let +## t = sol.t[end] +## u_ode = sol.u[end] +## du_ode = similar(u_ode) +## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 +## end +## end +## +## @trixi_testset "elixir_shallowwater_source_terms.jl" begin +## # This test is identical to the one in `test_p4est_2d.jl`. +## @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_shallowwater_source_terms.jl"), +## l2=[ +## 9.168126407325352e-5, +## 0.0009795410115453788, +## 0.002546408320320785, +## 3.941189812642317e-6, +## ], +## linf=[ +## 0.0009903782521019089, +## 0.0059752684687262025, +## 0.010941106525454103, +## 1.2129488214718265e-5, +## ], +## tspan=(0.0, 0.1)) +## # Ensure that we do not have excessive memory allocations +## # (e.g., from type instabilities) +## let +## t = sol.t[end] +## u_ode = sol.u[end] +## du_ode = similar(u_ode) +## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 +## end +## end +## +## @trixi_testset "elixir_mhd_alfven_wave.jl" begin +## # This test is identical to the one in `test_p4est_2d.jl`. +## @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_mhd_alfven_wave.jl"), +## l2=[1.0513414461545583e-5, 1.0517900957166411e-6, +## 1.0517900957304043e-6, 1.511816606372376e-6, +## 1.0443997728645063e-6, 7.879639064990798e-7, +## 7.879639065049896e-7, 1.0628631669056271e-6, +## 4.3382328912336153e-7], +## linf=[4.255466285174592e-5, 1.0029706745823264e-5, +## 1.0029706747467781e-5, 1.2122265939010224e-5, +## 5.4791097160444835e-6, 5.18922042269665e-6, +## 5.189220422141538e-6, 9.552667261422676e-6, +## 1.4237578427628152e-6]) +## # Ensure that we do not have excessive memory allocations +## # (e.g., from type instabilities) +## let +## t = sol.t[end] +## u_ode = sol.u[end] +## du_ode = similar(u_ode) +## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 +## end +## end +## +## @trixi_testset "elixir_mhd_rotor.jl" begin +## # This test is identical to the one in `test_p4est_2d.jl` besides minor +## # deviations in the expected error norms. +## @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_mhd_rotor.jl"), +## l2=[0.44211360369891683, 0.8805178316216257, 0.8262710688468049, +## 0.0, +## 0.9616090460973586, 0.10386643568745411, +## 0.15403457366543802, 0.0, +## 2.8399715649715473e-5], +## linf=[10.04369305341599, 17.995640564998403, 9.576041548174265, +## 0.0, +## 19.429658884314534, 1.3821395681242314, 1.818559351543182, +## 0.0, +## 0.002261930217575465], +## tspan=(0.0, 0.02)) +## # Ensure that we do not have excessive memory allocations +## # (e.g., from type instabilities) +## let +## t = sol.t[end] +## u_ode = sol.u[end] +## du_ode = similar(u_ode) +## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 +## end +## end end # Clean up afterwards: delete Trixi.jl output directory From d739b919bf365696aaffddf65103642de6d438bf Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Wed, 8 May 2024 15:23:54 +0200 Subject: [PATCH 39/81] Update src/meshes/t8code_mesh.jl Co-authored-by: Michael Schlottke-Lakemper --- src/meshes/t8code_mesh.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index bca0e62d9d9..3776380639f 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -197,7 +197,7 @@ function T8codeMesh{NDIMS, RealT}(forest::Ptr{t8_forest}, boundary_names; polyde vol = dot(cross(u, v), w) if vol < z - @warn "Discovered negative volumes in `cmesh`: vol = $vol" + error("Discovered negative volumes in `cmesh`: vol = $vol") end end From d69e4bf64d001291923c418cf5e14d64f9076cd2 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Wed, 8 May 2024 15:28:24 +0200 Subject: [PATCH 40/81] Appyling comments. --- src/auxiliary/auxiliary.jl | 9 ---- src/meshes/t8code_mesh.jl | 97 +++++++++++++++++++++----------------- 2 files changed, 54 insertions(+), 52 deletions(-) diff --git a/src/auxiliary/auxiliary.jl b/src/auxiliary/auxiliary.jl index f3c54249816..44bc16ee38d 100644 --- a/src/auxiliary/auxiliary.jl +++ b/src/auxiliary/auxiliary.jl @@ -371,13 +371,4 @@ function download(src_url, file_path) return file_path end -abstract type MeshFile{NDIMS} end - -struct GmshFile{NDIMS} <: MeshFile{NDIMS} - path::String -end - -struct AbaqusFile{NDIMS} <: MeshFile{NDIMS} - path::String -end end # @muladd diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 52e2ad06b22..ebab8cedab9 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -101,17 +101,19 @@ function Base.show(io::IO, ::MIME"text/plain", mesh::T8codeMesh) end """ - T8codeMesh{NDIMS, RealT}(forest, boundary_names; polydeg = 1, mapping=identity) + T8codeMesh{NDIMS, RealT}(forest, boundary_names; polydeg = 1, mapping = nothing) -Create a 'T8codeMesh'. +Main mesh constructor for the `T8codeMesh` wrapping around a given t8code +`forest` object. This constructor is typically called by other `T8codeMesh` +constructors. # Arguments -- `forest`: Pointer to a committed forest. +- `forest`: Pointer to a t8code forest. - `boundary_names`: List of boundary names. - `polydeg::Integer`: Polynomial degree used to store the geometry of the mesh. The mapping will be approximated by an interpolation polynomial of the specified degree for each tree. -- `mapping`: a function of `NDIMS` variables to describe the mapping that transforms +- `mapping`: A function of `NDIMS` variables to describe the mapping that transforms the imported mesh to the physical domain. Use `nothing` for the identity map. """ function T8codeMesh{NDIMS, RealT}(forest::Ptr{t8_forest}, boundary_names; polydeg = 1, @@ -121,35 +123,35 @@ function T8codeMesh{NDIMS, RealT}(forest::Ptr{t8_forest}, boundary_names; polyde nodes = 0.5 .* (basis.nodes .+ 1.0) cmesh = t8_forest_get_cmesh(forest) - num_trees = t8_forest_get_num_global_trees(forest) + number_of_trees = t8_forest_get_num_global_trees(forest) tree_node_coordinates = Array{RealT, NDIMS + 2}(undef, NDIMS, ntuple(_ -> length(nodes), NDIMS)..., - num_trees) + number_of_trees) - coords_ref = Vector{Float64}(undef, 3) + reference_coordinates = Vector{Float64}(undef, 3) # Calculate node coordinates of reference mesh. if NDIMS == 2 - num_corners = 4 # quadrilateral + number_of_corners = 4 # quadrilateral # Testing for negative element volumes. - verts = zeros(3, num_corners) - for itree in 1:num_trees - veptr = t8_cmesh_get_tree_vertices(cmesh, itree - 1) + vertices = zeros(3, number_of_corners) + for itree in 1:number_of_trees + vertices_pointer = t8_cmesh_get_tree_vertices(cmesh, itree - 1) - # Note, `verts = unsafe_wrap(Array, veptr, (3, 1 << NDIMS))` - # sometimes does not work since `veptr` is not necessarily properly + # Note, `vertices = unsafe_wrap(Array, vertices_pointer, (3, 1 << NDIMS))` + # sometimes does not work since `vertices_pointer` is not necessarily properly # aligned to 8 bytes. - for icorner in 1:num_corners - verts[1, icorner] = unsafe_load(veptr, (icorner - 1) * 3 + 1) - verts[2, icorner] = unsafe_load(veptr, (icorner - 1) * 3 + 2) + for icorner in 1:number_of_corners + vertices[1, icorner] = unsafe_load(vertices_pointer, (icorner - 1) * 3 + 1) + vertices[2, icorner] = unsafe_load(vertices_pointer, (icorner - 1) * 3 + 2) end # Check if tree's node ordering is right-handed or print a warning. - let z = zero(eltype(verts)), o = one(eltype(verts)) - u = verts[:, 2] - verts[:, 1] - v = verts[:, 3] - verts[:, 1] + let z = zero(eltype(vertices)), o = one(eltype(vertices)) + u = vertices[:, 2] - vertices[:, 1] + v = vertices[:, 3] - vertices[:, 1] w = [z, z, o] # Triple product gives signed volume of spanned parallelepiped. @@ -162,36 +164,36 @@ function T8codeMesh{NDIMS, RealT}(forest::Ptr{t8_forest}, boundary_names; polyde # Query geometry data from t8code. for j in eachindex(nodes), i in eachindex(nodes) - coords_ref[1] = nodes[i] - coords_ref[2] = nodes[j] - coords_ref[3] = 0.0 - t8_geometry_evaluate(cmesh, itree - 1, coords_ref, 1, + reference_coordinates[1] = nodes[i] + reference_coordinates[2] = nodes[j] + reference_coordinates[3] = 0.0 + t8_geometry_evaluate(cmesh, itree - 1, reference_coordinates, 1, @view(tree_node_coordinates[:, i, j, itree])) end end elseif NDIMS == 3 - num_corners = 8 # hexahedron + number_of_corners = 8 # hexahedron # Testing for negative element volumes. - verts = zeros(3, num_corners) - for itree in 1:num_trees - veptr = t8_cmesh_get_tree_vertices(cmesh, itree - 1) + vertices = zeros(3, number_of_corners) + for itree in 1:number_of_trees + vertices_pointer = t8_cmesh_get_tree_vertices(cmesh, itree - 1) - # Note, `verts = unsafe_wrap(Array, veptr, (3, 1 << NDIMS))` - # sometimes does not work since `veptr` is not necessarily properly + # Note, `vertices = unsafe_wrap(Array, vertices_pointer, (3, 1 << NDIMS))` + # sometimes does not work since `vertices_pointer` is not necessarily properly # aligned to 8 bytes. - for icorner in 1:num_corners - verts[1, icorner] = unsafe_load(veptr, (icorner - 1) * 3 + 1) - verts[2, icorner] = unsafe_load(veptr, (icorner - 1) * 3 + 2) - verts[3, icorner] = unsafe_load(veptr, (icorner - 1) * 3 + 3) + for icorner in 1:number_of_corners + vertices[1, icorner] = unsafe_load(vertices_pointer, (icorner - 1) * 3 + 1) + vertices[2, icorner] = unsafe_load(vertices_pointer, (icorner - 1) * 3 + 2) + vertices[3, icorner] = unsafe_load(vertices_pointer, (icorner - 1) * 3 + 3) end # Check if tree's node ordering is right-handed or print a warning. - let z = zero(eltype(verts)) - u = verts[:, 2] - verts[:, 1] - v = verts[:, 3] - verts[:, 1] - w = verts[:, 5] - verts[:, 1] + let z = zero(eltype(vertices)) + u = vertices[:, 2] - vertices[:, 1] + v = vertices[:, 3] - vertices[:, 1] + w = vertices[:, 5] - vertices[:, 1] # Triple product gives signed volume of spanned parallelepiped. vol = dot(cross(u, v), w) @@ -203,15 +205,15 @@ function T8codeMesh{NDIMS, RealT}(forest::Ptr{t8_forest}, boundary_names; polyde # Query geometry data from t8code. for k in eachindex(nodes), j in eachindex(nodes), i in eachindex(nodes) - coords_ref[1] = nodes[i] - coords_ref[2] = nodes[j] - coords_ref[3] = nodes[k] - t8_geometry_evaluate(cmesh, itree - 1, coords_ref, 1, + reference_coordinates[1] = nodes[i] + reference_coordinates[2] = nodes[j] + reference_coordinates[3] = nodes[k] + t8_geometry_evaluate(cmesh, itree - 1, reference_coordinates, 1, @view(tree_node_coordinates[:, i, j, k, itree])) end - else - throw(ArgumentError("$NDIMS dimensions are not supported.")) end + else + throw(ArgumentError("$NDIMS dimensions are not supported.")) end # Apply user defined mapping. @@ -429,6 +431,15 @@ function T8codeMesh(conn::Ptr{p8est_connectivity}; kwargs...) return T8codeMesh(cmesh; kwargs...) end +# Convenience types for multiple dispatch. Only used in this file. +struct GmshFile{NDIMS} + path::String +end + +struct AbaqusFile{NDIMS} + path::String +end + """ T8codeMesh(meshfile::String, NDIMS; kwargs...) From 1cec70e46bf5ed73d1ed84242e40664b6251b7bd Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Wed, 8 May 2024 15:38:33 +0200 Subject: [PATCH 41/81] Applied formatter. --- src/auxiliary/auxiliary.jl | 1 - test/test_t8code_2d.jl | 512 ++++++++++++++++++------------------- 2 files changed, 256 insertions(+), 257 deletions(-) diff --git a/src/auxiliary/auxiliary.jl b/src/auxiliary/auxiliary.jl index 44bc16ee38d..972a748c56b 100644 --- a/src/auxiliary/auxiliary.jl +++ b/src/auxiliary/auxiliary.jl @@ -370,5 +370,4 @@ function download(src_url, file_path) return file_path end - end # @muladd diff --git a/test/test_t8code_2d.jl b/test/test_t8code_2d.jl index f86747245f1..b63d2a105ac 100644 --- a/test/test_t8code_2d.jl +++ b/test/test_t8code_2d.jl @@ -39,7 +39,7 @@ end end @trixi_testset "test check_for_negative_volumes" begin - @test_warn "Discovered negative volumes" begin + @test_throws "Discovered negative volumes" begin # Unstructured mesh with six cells which have left-handed node ordering. mesh_file = Trixi.download("https://gist.githubusercontent.com/jmark/bfe0d45f8e369298d6cc637733819013/raw/cecf86edecc736e8b3e06e354c494b2052d41f7a/rectangle_with_negative_volumes.msh", joinpath(EXAMPLES_DIR, @@ -66,265 +66,265 @@ end # Unstructured ABAQUS mesh file created with HOHQMesh.. file_path = Trixi.download("https://gist.githubusercontent.com/jmark/9e0da4306e266617eeb19bc56b0e7feb/raw/e6856e1deb648a807f6bb6d6dcacff9e55d94e2a/round_2d_tank.inp", joinpath(EXAMPLES_DIR, "round_2d_tank.inp")) - mesh = T8codeMesh(AbaqusFile{2}(file_path)) + mesh = T8codeMesh(file_path, 2) all(size(mesh.tree_node_coordinates) .== (2, 4, 4, 340)) end end -## @trixi_testset "elixir_advection_basic.jl" begin -## # This test is identical to the one in `test_p4est_2d.jl`. -## @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_basic.jl"), -## # Expected errors are exactly the same as with TreeMesh! -## l2=[8.311947673061856e-6], -## linf=[6.627000273229378e-5]) -## # Ensure that we do not have excessive memory allocations -## # (e.g., from type instabilities) -## let -## t = sol.t[end] -## u_ode = sol.u[end] -## du_ode = similar(u_ode) -## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 -## end -## end -## -## @trixi_testset "elixir_advection_nonconforming_flag.jl" begin -## # This test is identical to the one in `test_p4est_2d.jl`. -## @test_trixi_include(joinpath(EXAMPLES_DIR, -## "elixir_advection_nonconforming_flag.jl"), -## l2=[3.198940059144588e-5], -## linf=[0.00030636069494005547]) -## # Ensure that we do not have excessive memory allocations -## # (e.g., from type instabilities) -## let -## t = sol.t[end] -## u_ode = sol.u[end] -## du_ode = similar(u_ode) -## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 -## end -## end -## -## @trixi_testset "elixir_advection_unstructured_flag.jl" begin -## # This test is identical to the one in `test_p4est_2d.jl`. -## @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_unstructured_flag.jl"), -## l2=[0.0005379687442422346], -## linf=[0.007438525029884735]) -## # Ensure that we do not have excessive memory allocations -## # (e.g., from type instabilities) -## let -## t = sol.t[end] -## u_ode = sol.u[end] -## du_ode = similar(u_ode) -## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 -## end -## end -## -## @trixi_testset "elixir_advection_amr_unstructured_flag.jl" begin -## @test_trixi_include(joinpath(EXAMPLES_DIR, -## "elixir_advection_amr_unstructured_flag.jl"), -## l2=[0.001993165013217687], -## linf=[0.032891018571625796], -## coverage_override=(maxiters = 6,)) -## # Ensure that we do not have excessive memory allocations -## # (e.g., from type instabilities) -## let -## t = sol.t[end] -## u_ode = sol.u[end] -## du_ode = similar(u_ode) -## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 -## end -## end -## -## @trixi_testset "elixir_advection_amr_solution_independent.jl" begin -## # This test is identical to the one in `test_p4est_2d.jl`. -## @test_trixi_include(joinpath(EXAMPLES_DIR, -## "elixir_advection_amr_solution_independent.jl"), -## # Expected errors are exactly the same as with StructuredMesh! -## l2=[4.949660644033807e-5], -## linf=[0.0004867846262313763], -## coverage_override=(maxiters = 6,)) -## # Ensure that we do not have excessive memory allocations -## # (e.g., from type instabilities) -## let -## t = sol.t[end] -## u_ode = sol.u[end] -## du_ode = similar(u_ode) -## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 -## end -## end -## -## @trixi_testset "elixir_euler_source_terms_nonconforming_unstructured_flag.jl" begin -## # This test is identical to the one in `test_p4est_2d.jl`. -## @test_trixi_include(joinpath(EXAMPLES_DIR, -## "elixir_euler_source_terms_nonconforming_unstructured_flag.jl"), -## l2=[ -## 0.0034516244508588046, -## 0.0023420334036925493, -## 0.0024261923964557187, -## 0.004731710454271893, -## ], -## linf=[ -## 0.04155789011775046, -## 0.024772109862748914, -## 0.03759938693042297, -## 0.08039824959535657, -## ]) -## # Ensure that we do not have excessive memory allocations -## # (e.g., from type instabilities) -## let -## t = sol.t[end] -## u_ode = sol.u[end] -## du_ode = similar(u_ode) -## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 -## end -## end -## -## @trixi_testset "elixir_euler_free_stream.jl" begin -## # This test is identical to the one in `test_p4est_2d.jl`. -## @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_euler_free_stream.jl"), -## l2=[ -## 2.063350241405049e-15, -## 1.8571016296925367e-14, -## 3.1769447886391905e-14, -## 1.4104095258528071e-14, -## ], -## linf=[1.9539925233402755e-14, 2e-12, 4.8e-12, 4e-12], -## atol=2.0e-12,) -## # Ensure that we do not have excessive memory allocations -## # (e.g., from type instabilities) -## let -## t = sol.t[end] -## u_ode = sol.u[end] -## du_ode = similar(u_ode) -## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 -## end -## end -## -## @trixi_testset "elixir_euler_shockcapturing_ec.jl" begin -## # This test is identical to the one in `test_p4est_2d.jl`. -## @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_euler_shockcapturing_ec.jl"), -## l2=[ -## 9.53984675e-02, -## 1.05633455e-01, -## 1.05636158e-01, -## 3.50747237e-01, -## ], -## linf=[ -## 2.94357464e-01, -## 4.07893014e-01, -## 3.97334516e-01, -## 1.08142520e+00, -## ], -## tspan=(0.0, 1.0)) -## # Ensure that we do not have excessive memory allocations -## # (e.g., from type instabilities) -## let -## t = sol.t[end] -## u_ode = sol.u[end] -## du_ode = similar(u_ode) -## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 -## end -## end -## -## @trixi_testset "elixir_euler_sedov.jl" begin -## # This test is identical to the one in `test_p4est_2d.jl` besides minor -## # deviations in the expected error norms. -## @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_euler_sedov.jl"), -## l2=[ -## 3.76149952e-01, -## 2.46970327e-01, -## 2.46970327e-01, -## 1.28889042e+00, -## ], -## linf=[ -## 1.22139001e+00, -## 1.17742626e+00, -## 1.17742626e+00, -## 6.20638482e+00, -## ], -## tspan=(0.0, 0.3)) -## # Ensure that we do not have excessive memory allocations -## # (e.g., from type instabilities) -## let -## t = sol.t[end] -## u_ode = sol.u[end] -## du_ode = similar(u_ode) -## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 -## end -## end -## -## @trixi_testset "elixir_shallowwater_source_terms.jl" begin -## # This test is identical to the one in `test_p4est_2d.jl`. -## @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_shallowwater_source_terms.jl"), -## l2=[ -## 9.168126407325352e-5, -## 0.0009795410115453788, -## 0.002546408320320785, -## 3.941189812642317e-6, -## ], -## linf=[ -## 0.0009903782521019089, -## 0.0059752684687262025, -## 0.010941106525454103, -## 1.2129488214718265e-5, -## ], -## tspan=(0.0, 0.1)) -## # Ensure that we do not have excessive memory allocations -## # (e.g., from type instabilities) -## let -## t = sol.t[end] -## u_ode = sol.u[end] -## du_ode = similar(u_ode) -## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 -## end -## end -## -## @trixi_testset "elixir_mhd_alfven_wave.jl" begin -## # This test is identical to the one in `test_p4est_2d.jl`. -## @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_mhd_alfven_wave.jl"), -## l2=[1.0513414461545583e-5, 1.0517900957166411e-6, -## 1.0517900957304043e-6, 1.511816606372376e-6, -## 1.0443997728645063e-6, 7.879639064990798e-7, -## 7.879639065049896e-7, 1.0628631669056271e-6, -## 4.3382328912336153e-7], -## linf=[4.255466285174592e-5, 1.0029706745823264e-5, -## 1.0029706747467781e-5, 1.2122265939010224e-5, -## 5.4791097160444835e-6, 5.18922042269665e-6, -## 5.189220422141538e-6, 9.552667261422676e-6, -## 1.4237578427628152e-6]) -## # Ensure that we do not have excessive memory allocations -## # (e.g., from type instabilities) -## let -## t = sol.t[end] -## u_ode = sol.u[end] -## du_ode = similar(u_ode) -## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 -## end -## end -## -## @trixi_testset "elixir_mhd_rotor.jl" begin -## # This test is identical to the one in `test_p4est_2d.jl` besides minor -## # deviations in the expected error norms. -## @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_mhd_rotor.jl"), -## l2=[0.44211360369891683, 0.8805178316216257, 0.8262710688468049, -## 0.0, -## 0.9616090460973586, 0.10386643568745411, -## 0.15403457366543802, 0.0, -## 2.8399715649715473e-5], -## linf=[10.04369305341599, 17.995640564998403, 9.576041548174265, -## 0.0, -## 19.429658884314534, 1.3821395681242314, 1.818559351543182, -## 0.0, -## 0.002261930217575465], -## tspan=(0.0, 0.02)) -## # Ensure that we do not have excessive memory allocations -## # (e.g., from type instabilities) -## let -## t = sol.t[end] -## u_ode = sol.u[end] -## du_ode = similar(u_ode) -## @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 -## end -## end +@trixi_testset "elixir_advection_basic.jl" begin + # This test is identical to the one in `test_p4est_2d.jl`. + @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_basic.jl"), + # Expected errors are exactly the same as with TreeMesh! + l2=[8.311947673061856e-6], + linf=[6.627000273229378e-5]) + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end +end + +@trixi_testset "elixir_advection_nonconforming_flag.jl" begin + # This test is identical to the one in `test_p4est_2d.jl`. + @test_trixi_include(joinpath(EXAMPLES_DIR, + "elixir_advection_nonconforming_flag.jl"), + l2=[3.198940059144588e-5], + linf=[0.00030636069494005547]) + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end +end + +@trixi_testset "elixir_advection_unstructured_flag.jl" begin + # This test is identical to the one in `test_p4est_2d.jl`. + @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_unstructured_flag.jl"), + l2=[0.0005379687442422346], + linf=[0.007438525029884735]) + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end +end + +@trixi_testset "elixir_advection_amr_unstructured_flag.jl" begin + @test_trixi_include(joinpath(EXAMPLES_DIR, + "elixir_advection_amr_unstructured_flag.jl"), + l2=[0.001993165013217687], + linf=[0.032891018571625796], + coverage_override=(maxiters = 6,)) + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end +end + +@trixi_testset "elixir_advection_amr_solution_independent.jl" begin + # This test is identical to the one in `test_p4est_2d.jl`. + @test_trixi_include(joinpath(EXAMPLES_DIR, + "elixir_advection_amr_solution_independent.jl"), + # Expected errors are exactly the same as with StructuredMesh! + l2=[4.949660644033807e-5], + linf=[0.0004867846262313763], + coverage_override=(maxiters = 6,)) + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end +end + +@trixi_testset "elixir_euler_source_terms_nonconforming_unstructured_flag.jl" begin + # This test is identical to the one in `test_p4est_2d.jl`. + @test_trixi_include(joinpath(EXAMPLES_DIR, + "elixir_euler_source_terms_nonconforming_unstructured_flag.jl"), + l2=[ + 0.0034516244508588046, + 0.0023420334036925493, + 0.0024261923964557187, + 0.004731710454271893, + ], + linf=[ + 0.04155789011775046, + 0.024772109862748914, + 0.03759938693042297, + 0.08039824959535657, + ]) + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end +end + +@trixi_testset "elixir_euler_free_stream.jl" begin + # This test is identical to the one in `test_p4est_2d.jl`. + @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_euler_free_stream.jl"), + l2=[ + 2.063350241405049e-15, + 1.8571016296925367e-14, + 3.1769447886391905e-14, + 1.4104095258528071e-14, + ], + linf=[1.9539925233402755e-14, 2e-12, 4.8e-12, 4e-12], + atol=2.0e-12,) + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end +end + +@trixi_testset "elixir_euler_shockcapturing_ec.jl" begin + # This test is identical to the one in `test_p4est_2d.jl`. + @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_euler_shockcapturing_ec.jl"), + l2=[ + 9.53984675e-02, + 1.05633455e-01, + 1.05636158e-01, + 3.50747237e-01, + ], + linf=[ + 2.94357464e-01, + 4.07893014e-01, + 3.97334516e-01, + 1.08142520e+00, + ], + tspan=(0.0, 1.0)) + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end +end + +@trixi_testset "elixir_euler_sedov.jl" begin + # This test is identical to the one in `test_p4est_2d.jl` besides minor + # deviations in the expected error norms. + @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_euler_sedov.jl"), + l2=[ + 3.76149952e-01, + 2.46970327e-01, + 2.46970327e-01, + 1.28889042e+00, + ], + linf=[ + 1.22139001e+00, + 1.17742626e+00, + 1.17742626e+00, + 6.20638482e+00, + ], + tspan=(0.0, 0.3)) + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end +end + +@trixi_testset "elixir_shallowwater_source_terms.jl" begin + # This test is identical to the one in `test_p4est_2d.jl`. + @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_shallowwater_source_terms.jl"), + l2=[ + 9.168126407325352e-5, + 0.0009795410115453788, + 0.002546408320320785, + 3.941189812642317e-6, + ], + linf=[ + 0.0009903782521019089, + 0.0059752684687262025, + 0.010941106525454103, + 1.2129488214718265e-5, + ], + tspan=(0.0, 0.1)) + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end +end + +@trixi_testset "elixir_mhd_alfven_wave.jl" begin + # This test is identical to the one in `test_p4est_2d.jl`. + @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_mhd_alfven_wave.jl"), + l2=[1.0513414461545583e-5, 1.0517900957166411e-6, + 1.0517900957304043e-6, 1.511816606372376e-6, + 1.0443997728645063e-6, 7.879639064990798e-7, + 7.879639065049896e-7, 1.0628631669056271e-6, + 4.3382328912336153e-7], + linf=[4.255466285174592e-5, 1.0029706745823264e-5, + 1.0029706747467781e-5, 1.2122265939010224e-5, + 5.4791097160444835e-6, 5.18922042269665e-6, + 5.189220422141538e-6, 9.552667261422676e-6, + 1.4237578427628152e-6]) + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end +end + +@trixi_testset "elixir_mhd_rotor.jl" begin + # This test is identical to the one in `test_p4est_2d.jl` besides minor + # deviations in the expected error norms. + @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_mhd_rotor.jl"), + l2=[0.44211360369891683, 0.8805178316216257, 0.8262710688468049, + 0.0, + 0.9616090460973586, 0.10386643568745411, + 0.15403457366543802, 0.0, + 2.8399715649715473e-5], + linf=[10.04369305341599, 17.995640564998403, 9.576041548174265, + 0.0, + 19.429658884314534, 1.3821395681242314, 1.818559351543182, + 0.0, + 0.002261930217575465], + tspan=(0.0, 0.02)) + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end +end end # Clean up afterwards: delete Trixi.jl output directory From 3158c4449f8902b49a26c812b021d37914cdcf4d Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Fri, 10 May 2024 10:01:03 +0200 Subject: [PATCH 42/81] Reverting changes to original state. --- src/solvers/dgsem_t8code/containers_2d.jl | 20 +++++++++---- src/solvers/dgsem_t8code/containers_3d.jl | 34 +++++++++++++++-------- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/src/solvers/dgsem_t8code/containers_2d.jl b/src/solvers/dgsem_t8code/containers_2d.jl index fb17b45d21e..ce525bfdf65 100644 --- a/src/solvers/dgsem_t8code/containers_2d.jl +++ b/src/solvers/dgsem_t8code/containers_2d.jl @@ -3,7 +3,7 @@ # we need to opt-in explicitly. # See https://ranocha.de/blog/Optimizing_EC_Trixi for further details. @muladd begin -#! format: off +#! format: noindent # Interpolate tree_node_coordinates to each quadrant at the specified nodes. function calc_node_coordinates!(node_coordinates, @@ -41,14 +41,22 @@ function calc_node_coordinates!(node_coordinates, t8_element_vertex_reference_coords(eclass_scheme, element, 0, pointer(element_coords)) - nodes_out_x = 2 * (element_length * 1 / 2 * (nodes .+ 1) .+ element_coords[1]) .- 1 - nodes_out_y = 2 * (element_length * 1 / 2 * (nodes .+ 1) .+ element_coords[2]) .- 1 + nodes_out_x = 2 * + (element_length * 1 / 2 * (nodes .+ 1) .+ element_coords[1]) .- + 1 + nodes_out_y = 2 * + (element_length * 1 / 2 * (nodes .+ 1) .+ element_coords[2]) .- + 1 - polynomial_interpolation_matrix!(matrix1, mesh.nodes, nodes_out_x, baryweights_in) - polynomial_interpolation_matrix!(matrix2, mesh.nodes, nodes_out_y, baryweights_in) + polynomial_interpolation_matrix!(matrix1, mesh.nodes, nodes_out_x, + baryweights_in) + polynomial_interpolation_matrix!(matrix2, mesh.nodes, nodes_out_y, + baryweights_in) multiply_dimensionwise!(view(node_coordinates, :, :, :, current_index += 1), - matrix1, matrix2, view(mesh.tree_node_coordinates, :, :, :, global_itree + 1), + matrix1, matrix2, + view(mesh.tree_node_coordinates, :, :, :, + global_itree + 1), tmp1) end end diff --git a/src/solvers/dgsem_t8code/containers_3d.jl b/src/solvers/dgsem_t8code/containers_3d.jl index 364877a09cf..4d56bc734aa 100644 --- a/src/solvers/dgsem_t8code/containers_3d.jl +++ b/src/solvers/dgsem_t8code/containers_3d.jl @@ -3,7 +3,7 @@ # we need to opt-in explicitly. # See https://ranocha.de/blog/Optimizing_EC_Trixi for further details. @muladd begin -#! format: off +#! format: noindent # Interpolate tree_node_coordinates to each quadrant at the specified nodes function calc_node_coordinates!(node_coordinates, @@ -43,17 +43,29 @@ function calc_node_coordinates!(node_coordinates, t8_element_vertex_reference_coords(eclass_scheme, element, 0, pointer(element_coords)) - nodes_out_x = (2 * (element_length * 0.5 * (nodes .+ 1) .+ element_coords[1]) .- 1) - nodes_out_y = (2 * (element_length * 0.5 * (nodes .+ 1) .+ element_coords[2]) .- 1) - nodes_out_z = (2 * (element_length * 0.5 * (nodes .+ 1) .+ element_coords[3]) .- 1) - - polynomial_interpolation_matrix!(matrix1, mesh.nodes, nodes_out_x, baryweights_in) - polynomial_interpolation_matrix!(matrix2, mesh.nodes, nodes_out_y, baryweights_in) - polynomial_interpolation_matrix!(matrix3, mesh.nodes, nodes_out_z, baryweights_in) - - multiply_dimensionwise!(view(node_coordinates, :, :, :, :, current_index += 1), + nodes_out_x = (2 * + (element_length * 0.5 * (nodes .+ 1) .+ element_coords[1]) .- + 1) + nodes_out_y = (2 * + (element_length * 0.5 * (nodes .+ 1) .+ element_coords[2]) .- + 1) + nodes_out_z = (2 * + (element_length * 0.5 * (nodes .+ 1) .+ element_coords[3]) .- + 1) + + polynomial_interpolation_matrix!(matrix1, mesh.nodes, nodes_out_x, + baryweights_in) + polynomial_interpolation_matrix!(matrix2, mesh.nodes, nodes_out_y, + baryweights_in) + polynomial_interpolation_matrix!(matrix3, mesh.nodes, nodes_out_z, + baryweights_in) + + multiply_dimensionwise!(view(node_coordinates, :, :, :, :, + current_index += 1), matrix1, matrix2, matrix3, - view(mesh.tree_node_coordinates, :, :, :, :, global_itree + 1), tmp1) + view(mesh.tree_node_coordinates, :, :, :, :, + global_itree + 1), + tmp1) end end From c706e6b11d64fae903046a4c4afc91b6774ea3ba Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Fri, 10 May 2024 16:18:21 +0200 Subject: [PATCH 43/81] Switching to ThreeEquation naming. --- Project.toml | 2 +- src/Trixi.jl | 6 +- src/equations/equations.jl | 4 + src/equations/three_equations_2d.jl | 545 ++++++++++++++++++++++++ src/solvers/dgsem_tree/dg_2d.jl | 52 ++- src/solvers/dgsem_tree/indicators.jl | 47 ++ src/solvers/dgsem_tree/indicators_2d.jl | 37 ++ 7 files changed, 683 insertions(+), 10 deletions(-) create mode 100644 src/equations/three_equations_2d.jl diff --git a/Project.toml b/Project.toml index 812d01733a4..bc63eff178b 100644 --- a/Project.toml +++ b/Project.toml @@ -92,7 +92,7 @@ StaticArrays = "1.5" StrideArrays = "0.1.26" StructArrays = "0.6.11" SummationByPartsOperators = "0.5.41" -T8code = "0.5" +T8code = "0.6" TimerOutputs = "0.5.7" Triangulate = "2.2" TriplotBase = "0.1" diff --git a/src/Trixi.jl b/src/Trixi.jl index f3977f1f058..fe8bbf785d7 100644 --- a/src/Trixi.jl +++ b/src/Trixi.jl @@ -162,6 +162,7 @@ export AcousticPerturbationEquations2D, ShallowWaterEquationsQuasi1D, LinearizedEulerEquations1D, LinearizedEulerEquations2D, LinearizedEulerEquations3D, PolytropicEulerEquations2D, + ThreeEquations2D, TrafficFlowLWREquations1D export LaplaceDiffusion1D, LaplaceDiffusion2D, LaplaceDiffusion3D, @@ -187,6 +188,7 @@ export flux, flux_central, flux_lax_friedrichs, flux_hll, flux_hllc, flux_hlle, FluxRotated, flux_shima_etal_turbo, flux_ranocha_turbo, FluxHydrostaticReconstruction, + flux_nonconservative_ThreeEquations_well, FluxUpwind export splitting_steger_warming, splitting_vanleer_haenel, @@ -218,7 +220,7 @@ export initial_condition_eoc_test_coupled_euler_gravity, export cons2cons, cons2prim, prim2cons, cons2macroscopic, cons2state, cons2mean, cons2entropy, entropy2cons export density, pressure, density_pressure, velocity, global_mean_vars, - equilibrium_distribution, waterheight_pressure + equilibrium_distribution, waterheight_pressure, alpha_rho export entropy, energy_total, energy_kinetic, energy_internal, energy_magnetic, cross_helicity, enstrophy @@ -270,7 +272,7 @@ export load_mesh, load_time, load_timestep, load_timestep!, load_dt, load_adaptive_time_integrator! export ControllerThreeLevel, ControllerThreeLevelCombined, - IndicatorLöhner, IndicatorLoehner, IndicatorMax + IndicatorLöhner, IndicatorLoehner, IndicatorMax, IndicatorClamp export PositivityPreservingLimiterZhangShu diff --git a/src/equations/equations.jl b/src/equations/equations.jl index 69a54f7faf8..f98bc01c9d5 100644 --- a/src/equations/equations.jl +++ b/src/equations/equations.jl @@ -508,6 +508,10 @@ include("linearized_euler_3d.jl") abstract type AbstractEquationsParabolic{NDIMS, NVARS, GradientVariables} <: AbstractEquations{NDIMS, NVARS} end +abstract type AbstractThreeEquationsEquations{NDIMS, NVARS} <: + AbstractEquations{NDIMS, NVARS} end +include("three_equations_2d.jl") + # Lighthill-Witham-Richards (LWR) traffic flow model abstract type AbstractTrafficFlowLWREquations{NDIMS, NVARS} <: AbstractEquations{NDIMS, NVARS} end diff --git a/src/equations/three_equations_2d.jl b/src/equations/three_equations_2d.jl new file mode 100644 index 00000000000..7c47236f091 --- /dev/null +++ b/src/equations/three_equations_2d.jl @@ -0,0 +1,545 @@ +# By default, Julia/LLVM does not use fused multiply-add operations (FMAs). +# Since these FMAs can increase the performance of many numerical algorithms, +# we need to opt-in explicitly. +# See https://ranocha.de/blog/Optimizing_EC_Trixi for further details. +@muladd begin + @doc raw""" + ThreeEquations2D(gamma) + + The three equations model for two phase flows. + + """ + struct ThreeEquations2D{RealT <: Real} <: AbstractThreeEquationsEquations{2, 6} + gamma::RealT + k0::RealT + rho_0::RealT + gravity::RealT + + function ThreeEquations2D(gamma, k0, rho_0, gravity) + return new{typeof(gamma)}(gamma, k0, rho_0, gravity) + end + end + + have_nonconservative_terms(::ThreeEquations2D) = True() + function varnames(::typeof(cons2cons), ::ThreeEquations2D) + ("alpha_rho", "alpha_rho_v1", "alpha_rho_v2", "alpha", "phi", "dummy") + end + varnames(::typeof(cons2prim), ::ThreeEquations2D) = ("rho", "v1", "v2", "alpha", "phi", "dummy") + varnames(::typeof(cons2entropy), ::ThreeEquations2D) = ("rho", "v1", "v2", "alpha", "phi", "dummy") + + n_nonconservative_terms(::ThreeEquations2D) = 1 + + # Set initial conditions at physical location `x` for time `t` + """ + initial_condition_constant(x, t, equations::ThreeEquations2D) + + A constant initial condition to test free-stream preservation. + """ + function initial_condition_constant(x, t, equations::ThreeEquations2D) + alpha_rho = 1000.0 + alpha_rho_v1 = 0.0 + alpha_rho_v2 = 0.0 + alpha = 1.0 + phi = x[2] + return SVector(alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi,0.0) + end + + function source_terms_gravity(u, x, t, equations::ThreeEquations2D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha = u + du1 = 0.0 + + period = 1.5 + amplitude = 0.05 + + du2 = -alpha_rho * 4*pi^2*amplitude/period^2*cos(2*pi*t / period) + # du3 = -alpha_rho * equations.gravity + du3 = 0.0 + + # if alpha > 2e-3 + # du2 = -alpha_rho * 5.0 + # du2 = 0.0 + # else + # du2 = 0.0 + # du3 = 0.0 + # end + + # du4 = abs(min(alpha-1e-3,0.0)) + du4 = 0.0 + du5 = 0.0 + + return SVector(du1, du2, du3, du4, du5,0.0) + end + + # function boundary_condition_wall(u_inner, orientation, + # direction, x, t, + # surface_flux_function, + # equations::ThreeEquations2D) + + # # Boundary state is equal to the inner state except for the velocity. For boundaries + # # in the -x/+x direction, we multiply the velocity in the x direction by -1. + # # Similarly, for boundaries in the -y/+y direction, we multiply the velocity in the + # # y direction by -1 + # # if direction in (1, 2) # x direction + # if orientation == 1 + # u_boundary = SVector(u_inner[1], -u_inner[2], u_inner[3], u_inner[4], + # u_inner[5]) + # else # y direction + # u_boundary = SVector(u_inner[1], u_inner[2], -u_inner[3], u_inner[4], + # u_inner[5]) + # end + + # # flux = surface_flux_function(u_inner, u_boundary, orientation, equations) + + # # Calculate boundary flux + # if iseven(direction) # u_inner is "left" of boundary, u_boundary is "right" of boundary + # flux = surface_flux_function(u_inner, u_boundary, orientation, equations) + # else # u_boundary is "left" of boundary, u_inner is "right" of boundary + # flux = surface_flux_function(u_boundary, u_inner, orientation, equations) + # end + + # # println(direction) + # # println(surface_flux_function) + # # println(pressure(u_inner, equations)) + # # println(pressure(u_boundary, equations)) + # # println(u_inner) + # # println(u_boundary) + # # println(flux) + # # println("") + + # return flux + # end + + function boundary_condition_wall(u_inner, orientation, + direction, x, t, + surface_flux_function, + equations::ThreeEquations2D) + + if string(surface_flux_function) == "flux_nonconservative_ThreeEquations_well" + # Boundary state is equal to the inner state except for the velocity. For boundaries + # in the -x/+x direction, we multiply the velocity in the x direction by -1. + # Similarly, for boundaries in the -y/+y direction, we multiply the velocity in the + # y direction by -1 + # if direction in (1, 2) # x direction + # if orientation == 1 + # u_boundary = SVector(u_inner[1], -u_inner[2], u_inner[3], u_inner[4], + # u_inner[5]) + # else # y direction + # u_boundary = SVector(u_inner[1], u_inner[2], -u_inner[3], u_inner[4], + # u_inner[5]) + # end + + # # flux = surface_flux_function(u_inner, u_boundary, orientation, equations) + + # # Calculate boundary flux + # if iseven(direction) # u_inner is "left" of boundary, u_boundary is "right" of boundary + # flux = surface_flux_function(u_inner, u_boundary, orientation, equations) + # else # u_boundary is "left" of boundary, u_inner is "right" of boundary + # flux = surface_flux_function(u_boundary, u_inner, orientation, equations) + # end + + z = zero(eltype(u_inner)) + + flux = SVector(z, z, z, z, z, z) + + else + + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi = u_inner + # v1 = alpha_rho_v1 / alpha_rho + # v2 = alpha_rho_v2 / alpha_rho + + p = alpha * pressure(u_inner, equations) + + z = zero(eltype(u_inner)) + + # Calculate boundary flux + if orientation == 1 + flux = SVector(z, p, z, z, z, z) + else + flux = SVector(z, z, p, z, z, z) + end + + # println(direction) + # println(surface_flux_function) + # println(Symbol(surface_flux_function)) + # println(pressure(u_inner, equations)) + # println(u_inner) + # println(flux) + # println("") + + end + + return flux + end + + @inline function boundary_condition_wall(u_inner, normal_direction::AbstractVector, + x, t, + surface_flux_function, + equations::ThreeEquations2D) + + # Normalize the outward pointing direction. + normal = normal_direction / norm(normal_direction) + + # compute the normal velocity + u_normal = normal[1] * u_inner[2] + normal[2] * u_inner[3] + + period = 1.5 + amplitude = 0.05 + + # u_wall = u_inner[1] * 0.5 * sin(2*pi*t/1.0) + # u_wall = u_inner[1] * 2*pi*amplitude/period*sin(2*pi*t / period) + u_wall = 0.0 + + # create the "external" boundary solution state + u_boundary = SVector(u_inner[1], + u_inner[2] - 2.0 * u_normal * normal[1] + u_wall * abs(normal[1]), + u_inner[3] - 2.0 * u_normal * normal[2] + u_wall * abs(normal[2]), + u_inner[4], + u_inner[5], + u_inner[6]) + + # println("u_inner = ") + # display(u_inner) + + # println("u_boundary = ") + # display(u_boundary) + + # calculate the boundary flux + flux = surface_flux_function(u_inner, u_boundary, normal_direction, equations) + + return flux + end + + # @inline function boundary_condition_wall(u_inner, normal_direction::AbstractVector, + # x, t, + # surface_flux_function, + # equations::ThreeEquations2D) + # + # if string(surface_flux_function) == "flux_nonconservative_ThreeEquations_well" + + # z = zero(eltype(u_inner)) + # flux = SVector(z, z, z, z, z) + # return flux + + # end + + # alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi = u_inner + + # normal = normal_direction / norm(normal_direction) + # # compute the normal velocity + # # u_normal = normal_direction[1] * u_inner[2] + normal_direction[2] * u_inner[3] + # + # p = alpha * pressure(u_inner, equations) + # z = zero(eltype(u_inner)) + + # flux = SVector(z, normal[1]*p, normal[2]*p, z, z) + + # return flux + # end + + # Calculate 1D flux for a single point + @inline function flux(u, orientation::Integer, equations::ThreeEquations2D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi = u + v1 = alpha_rho_v1 / alpha_rho + v2 = alpha_rho_v2 / alpha_rho + p = pressure(u, equations) + if orientation == 1 + f1 = alpha_rho_v1 + f2 = alpha_rho_v1 * v1 + alpha * p + f3 = alpha_rho_v1 * v2 + f4 = 0.0 + f5 = 0.0 + f6 = 0.0 + else + f1 = alpha_rho_v2 + f2 = alpha_rho_v1 * v2 + f3 = alpha_rho_v2 * v2 + alpha * p + f4 = 0.0 + f5 = 0.0 + f6 = 0.0 + end + return SVector(f1, f2, f3, f4, f5, f6) + end + + # Calculate 1D flux for a single point in the normal direction + # Note, this directional vector is not normalized + @inline function flux(u, normal_direction::AbstractVector, equations::ThreeEquations2D) + rho, v1, v2, alpha, phi = cons2prim(u, equations) + + v_normal = v1 * normal_direction[1] + v2 * normal_direction[2] + rho_v_normal = rho * v_normal + + p = pressure(u, equations) + + f1 = alpha * rho_v_normal + f2 = alpha * rho_v_normal * v1 + alpha * p * normal_direction[1] + f3 = alpha * rho_v_normal * v2 + alpha * p * normal_direction[2] + f4 = 0.0 + f5 = 0.0 + f6 = 0.0 + + return SVector(f1, f2, f3, f4, f5, f6) + end + + @inline function flux_nonconservative_ThreeEquations(u_ll, u_rr, orientation::Integer, + equations::ThreeEquations2D) + v1_ll = u_ll[2] / u_ll[1] + v2_ll = u_ll[3] / u_ll[1] + alpha_rr = u_rr[4] + + z = zero(eltype(u_ll)) + + if orientation == 1 + f = SVector(z, z, z, v1_ll * alpha_rr, z, z) + else + f = SVector(z, z, z, v2_ll * alpha_rr, z, z) + end + + return f + end + + @inline function flux_nonconservative_ThreeEquations(u_ll, u_rr, + normal_direction_ll::AbstractVector, + normal_direction_average::AbstractVector, + equations::ThreeEquations2D) + v1_ll = u_ll[2] / u_ll[1] + v2_ll = u_ll[3] / u_ll[1] + alpha_rr = u_rr[4] + + v_dot_n_ll = v1_ll * normal_direction_ll[1] + v2_ll * normal_direction_ll[2] + + z = zero(eltype(u_ll)) + + f = SVector(z, z, z, v_dot_n_ll * alpha_rr, z, z) + + return f + end + + @inline function flux_nonconservative_ThreeEquations_well(u_ll, u_rr, orientation::Integer, + equations::ThreeEquations2D) + v1_ll = u_ll[2] / u_ll[1] + v2_ll = u_ll[3] / u_ll[1] + alpha_rr = u_rr[4] + phi_ll = u_ll[5] + phi_rr = u_rr[5] + + well_balanced = -u_ll[1] / equations.rho_0 * equations.k0 * + exp( equations.rho_0 * phi_ll / equations.k0) * + exp(-equations.rho_0 * phi_rr / equations.k0) + + z = zero(eltype(u_ll)) + + if orientation == 1 + f = SVector(z, well_balanced, z, v1_ll * alpha_rr, z, z) + else + f = SVector(z, z, well_balanced, v2_ll * alpha_rr, z, z) + end + + return f + end + + @inline function flux_nonconservative_ThreeEquations_well(u_ll, u_rr, + normal_direction_ll::AbstractVector, + normal_direction_average::AbstractVector, + equations::ThreeEquations2D) + v1_ll = u_ll[2] / u_ll[1] + v2_ll = u_ll[3] / u_ll[1] + alpha_rr = u_rr[4] + + phi_ll = u_ll[5] + phi_rr = u_rr[5] + + v_dot_n_ll = v1_ll * normal_direction_ll[1] + v2_ll * normal_direction_ll[2] + + # Non well-balanced force contribution. + force = -u_ll[1] * phi_rr + + # Well-balanced force contribution. + # force = -u_ll[1] * (equations.k0/equations.rho_0) * + # exp(-equations.rho_0/equations.k0 * phi_ll) * + # exp( equations.rho_0/equations.k0 * phi_rr) + + z = zero(eltype(u_ll)) + f = SVector(z, force * normal_direction_average[1], force * normal_direction_average[2], v_dot_n_ll * alpha_rr, z, z) + + return f + end + + # Calculate maximum wave speed for local Lax-Friedrichs-type dissipation as the + # maximum velocity magnitude plus the maximum speed of sound + @inline function max_abs_speed_naive(u_ll, u_rr, orientation::Integer, + equations::ThreeEquations2D) + rho_ll, v1_ll, v2_ll, alpha_ll, phi_ll, dummy_ll = cons2prim(u_ll, equations) + rho_rr, v1_rr, v2_rr, alpha_rr, phi_rr, dummy_rr = cons2prim(u_rr, equations) + + # Get the velocity value in the appropriate direction + if orientation == 1 + v_ll = v1_ll + v_rr = v1_rr + else # orientation == 2 + v_ll = v2_ll + v_rr = v2_rr + end + # Calculate sound speeds + c_ll = sqrt(equations.gamma * (equations.k0 / equations.rho_0) * + (rho_ll / equations.rho_0)^(equations.gamma - 1)) + c_rr = sqrt(equations.gamma * (equations.k0 / equations.rho_0) * + (rho_rr / equations.rho_0)^(equations.gamma - 1)) + + λ_max = max(abs(v_ll), abs(v_rr)) + max(c_ll, c_rr) + end + + @inline function max_abs_speed_naive(u_ll, u_rr, normal_direction::AbstractVector, + equations::ThreeEquations2D) + rho_ll, v1_ll, v2_ll, alpha_ll, phi_ll, dummy_ll = cons2prim(u_ll, equations) + rho_rr, v1_rr, v2_rr, alpha_rr, phi_rr, dummy_rr = cons2prim(u_rr, equations) + + # Calculate normal velocities and sound speed + # left + v_ll = (v1_ll * normal_direction[1] + + + v2_ll * normal_direction[2]) + c_ll = sqrt(equations.gamma * (equations.k0 / equations.rho_0) * + (rho_ll / equations.rho_0)^(equations.gamma - 1)) + # right + v_rr = (v1_rr * normal_direction[1] + + + v2_rr * normal_direction[2]) + c_rr = sqrt(equations.gamma * (equations.k0 / equations.rho_0) * + (rho_rr / equations.rho_0)^(equations.gamma - 1)) + + return max(abs(v_ll), abs(v_rr)) + max(c_ll, c_rr) * norm(normal_direction) + end + + # Calculate minimum and maximum wave speeds for HLL-type fluxes + @inline function min_max_speed_naive(u_ll, u_rr, orientation::Integer, + equations::ThreeEquations2D) + rho_ll, v1_ll, v2_ll, alpha_ll, phi_ll, dummy_ll = cons2prim(u_ll, equations) + rho_rr, v1_rr, v2_rr, alpha_rr, phi_rr, dummy_rr = cons2prim(u_rr, equations) + + if orientation == 1 # x-direction + λ_min = v1_ll - sqrt(equations.gamma * (equations.k0 / equations.rho_0) * + (rho_ll / equations.rho_0)^(equations.gamma - 1)) + λ_max = v1_rr + sqrt(equations.gamma * (equations.k0 / equations.rho_0) * + (rho_rr / equations.rho_0)^(equations.gamma - 1)) + else # y-direction + λ_min = v2_ll - sqrt(equations.gamma * (equations.k0 / equations.rho_0) * + (rho_ll / equations.rho_0)^(equations.gamma - 1)) + λ_max = v2_rr + sqrt(equations.gamma * (equations.k0 / equations.rho_0) * + (rho_rr / equations.rho_0)^(equations.gamma - 1)) + end + + return λ_min, λ_max + end + + @inline function min_max_speed_naive(u_ll, u_rr, normal_direction::AbstractVector, + equations::ThreeEquations2D) + rho_ll, v1_ll, v2_ll, alpha_ll, phi_ll, dummy_ll = cons2prim(u_ll, equations) + rho_rr, v1_rr, v2_rr, alpha_rr, phi_rr, dummy_rr = cons2prim(u_rr, equations) + + v_normal_ll = v1_ll * normal_direction[1] + v2_ll * normal_direction[2] + v_normal_rr = v1_rr * normal_direction[1] + v2_rr * normal_direction[2] + + norm_ = norm(normal_direction) + # The v_normals are already scaled by the norm + λ_min = v_normal_ll - + sqrt(equations.gamma * (equations.k0 / equations.rho_0) * + (rho_ll / equations.rho_0)^(equations.gamma - 1)) * norm_ + λ_max = v_normal_rr + + sqrt(equations.gamma * (equations.k0 / equations.rho_0) * + (rho_rr / equations.rho_0)^(equations.gamma - 1)) * norm_ + + return λ_min, λ_max + end + + @inline function max_abs_speeds(u, equations::ThreeEquations2D) + rho, v1, v2, alpha, phi, dummy = cons2prim(u, equations) + c = sqrt(equations.gamma * (equations.k0 / equations.rho_0) * + (rho / equations.rho_0)^(equations.gamma - 1)) + + return abs(v1) + c, abs(v2) + c + end + + # Convert conservative variables to primitive + @inline function cons2prim(u, equations::ThreeEquations2D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi, dummy = u + + rho = alpha_rho / alpha + v1 = alpha_rho_v1 / alpha_rho + v2 = alpha_rho_v2 / alpha_rho + + return SVector(rho, v1, v2, alpha, phi, dummy) + end + + # Convert conservative variables to primitive + @inline function cons2entropy(u, equations::ThreeEquations2D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi, dummy = u + + rho = alpha_rho / alpha + v1 = alpha_rho_v1 / alpha_rho + v2 = alpha_rho_v2 / alpha_rho + + phi = max(1e-6, max(max_abs_speeds(u, equations)...) ) + + # vn = sqrt(v1*v1 + v2*v2) + # vn = log10(sqrt(alpha_rho_v1^2 + alpha_rho_v2^2)) + vn = log10(max(1e-6, sqrt(v1^2 + v2^2))) + + alpha_log = log10(max(1e-6, alpha)) + p = pressure(u, equations) + + # return SVector(alpha_rho, vn, log10(alpha), alpha, phi) + # return SVector(alpha_rho, vn, alpha_log, alpha, phi) + return SVector(alpha_rho, p, alpha_log, alpha, phi, dummy) + end + + # Convert primitive to conservative variables + @inline function prim2cons(prim, equations::ThreeEquations2D) + rho, v1, v2, alpha, phi, dummy = prim + alpha_rho = rho * alpha + alpha_rho_v1 = alpha_rho * v1 + alpha_rho_v2 = alpha_rho * v2 + return SVector(alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi, dummy) + end + + @inline function isvalid(u, equations::ThreeEquations2D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi, dummy = u + # return alpha_rho > 0.0 && alpha > 0.0 && alpha <= 1.0 + return alpha_rho > 0.0 && alpha > 0.0 # && alpha <= 1.0 + end + + @inline function density(u, equations::ThreeEquations2D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi, dummy = u + return alpha_rho / alpha + end + + @inline function alpha_rho(u, equations::ThreeEquations2D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi, dummy = u + return alpha_rho + end + + @inline function alpha(u, equations::ThreeEquations2D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi, dummy = u + return alpha + end + + @inline function pressure(u, equations::ThreeEquations2D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi, dummy = u + # return alpha * equations.k0 * ((alpha_rho / alpha / equations.rho_0)^(equations.gamma) - 1) + return equations.k0 * ((alpha_rho / alpha / equations.rho_0)^(equations.gamma) - 1) + end + + @inline function density_pressure(u, equations::ThreeEquations2D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi, dummy = u + rho = alpha_rho / alpha + rho_times_p = pressure(u, equations) * rho + return rho_times_p + end + + # Calculate the error for the "water-at-rest" test case + @inline function water_at_rest_error(u, equations::ThreeEquations2D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi, dummy = u + rho0 = equations.rho_0 * + exp(-(equations.gravity * equations.rho_0 / equations.k0) * (phi - 1.0)) + return abs(alpha_rho / alpha - rho0) + end +end # @muladd diff --git a/src/solvers/dgsem_tree/dg_2d.jl b/src/solvers/dgsem_tree/dg_2d.jl index 6b66c2d4bfa..a9bc217aa1e 100644 --- a/src/solvers/dgsem_tree/dg_2d.jl +++ b/src/solvers/dgsem_tree/dg_2d.jl @@ -1151,20 +1151,58 @@ function calc_sources!(du, u, t, source_terms::Nothing, return nothing end +# function calc_sources!(du, u, t, source_terms, +# equations::AbstractEquations{2}, dg::DG, cache) +# @unpack node_coordinates = cache.elements +# +# @threaded for element in eachelement(dg, cache) +# for j in eachnode(dg), i in eachnode(dg) +# u_local = get_node_vars(u, equations, dg, i, j, element) +# x_local = get_node_coords(node_coordinates, equations, dg, +# i, j, element) +# du_local = source_terms(u_local, x_local, t, equations) +# add_to_node_vars!(du, du_local, equations, dg, i, j, element) +# end +# end +# +# return nothing +# end + function calc_sources!(du, u, t, source_terms, equations::AbstractEquations{2}, dg::DG, cache) @unpack node_coordinates = cache.elements @threaded for element in eachelement(dg, cache) - for j in eachnode(dg), i in eachnode(dg) - u_local = get_node_vars(u, equations, dg, i, j, element) - x_local = get_node_coords(node_coordinates, equations, dg, - i, j, element) - du_local = source_terms(u_local, x_local, t, equations) - add_to_node_vars!(du, du_local, equations, dg, i, j, element) - end + source_terms(du, u, node_coordinates, t, element, equations, dg) + + # N = length(dg.basis.nodes) + # u_local = Array{eltype(u),3}(undef, nvariables(equations), N, N) + # x_local = Array{eltype(u),3}(undef, 2, N, N) + # z = zero(eltype(u)) + + # for j in eachnode(dg), i in eachnode(dg) + # u_local[:,i,j] = get_node_vars(u, equations, dg, i, j, element) + # x_local[:,i,j] = get_node_coords(node_coordinates, equations, dg, + # i, j, element) + # end + + # # du_local = source_terms(u_local, x_local, t, equations, dg) + # # display(du_local) + + # for j in eachnode(dg), i in eachnode(dg) + + # # du[3,:,:] .= -u[1,:,:] * 9.81 + # # sv = SVector(du_local[:,i,j]...) + + # sv = SVector(z, z, -u_local[1,i,j] * 9.81, z, z) + # # println(sv) + # # add_to_node_vars!(du, @view(du_local[:,i,j]), equations, dg, i, j, element) + # # add_to_node_vars!(du, du_local[:,i,j], equations, dg, i, j, element) + # add_to_node_vars!(du, sv, equations, dg, i, j, element) + # end end return nothing end + end # @muladd diff --git a/src/solvers/dgsem_tree/indicators.jl b/src/solvers/dgsem_tree/indicators.jl index 9f25a6d2dbb..fa04359419b 100644 --- a/src/solvers/dgsem_tree/indicators.jl +++ b/src/solvers/dgsem_tree/indicators.jl @@ -256,4 +256,51 @@ function Base.show(io::IO, ::MIME"text/plain", indicator::IndicatorMax) summary_box(io, "IndicatorMax", setup) end end + +""" + IndicatorClamp(equations::AbstractEquations, basis; min=0.0, max=1.0, variable) + IndicatorClamp(semi::AbstractSemidiscretization; min=0.0, max=1.0, variable) + +A simple indicator returning 1.0 when the element average of `variable` lies within [min,max]. +Returns -1.0 otherwise. +""" +struct IndicatorClamp{RealT<:Real, Variable, Cache<:NamedTuple} <: AbstractIndicator + min::RealT + max::RealT + variable::Variable + cache::Cache +end + +function IndicatorClamp(equations::AbstractEquations, basis; min = 0.0, max = 1.0, variable) + cache = create_cache(IndicatorClamp, equations, basis) + IndicatorClamp{typeof(min), typeof(variable), typeof(cache)}(min, max, variable, cache) +end + +function IndicatorClamp(semi::AbstractSemidiscretization; min = 0.0, max = 1.0, variable) + cache = create_cache(IndicatorClamp, semi) + return IndicatorClamp{typeof(min), typeof(variable), typeof(cache)}(min, max, variable, cache) +end + +function Base.show(io::IO, indicator::IndicatorClamp) + @nospecialize indicator # reduce precompilation time + + print(io, "IndicatorClamp(") + print(io, "min=", indicator.min, ", max=", indicator.max, ", variable=", indicator.variable, ")") +end + +function Base.show(io::IO, ::MIME"text/plain", indicator::IndicatorClamp) + @nospecialize indicator # reduce precompilation time + + if get(io, :compact, false) + show(io, indicator) + else + setup = [ + "indicator variable" => indicator.variable, + "min" => indicator.min, + "max" => indicator.max, + ] + summary_box(io, "IndicatorClamp", setup) + end +end + end # @muladd diff --git a/src/solvers/dgsem_tree/indicators_2d.jl b/src/solvers/dgsem_tree/indicators_2d.jl index 665d2254e5d..e6629c56449 100644 --- a/src/solvers/dgsem_tree/indicators_2d.jl +++ b/src/solvers/dgsem_tree/indicators_2d.jl @@ -229,4 +229,41 @@ function (indicator_max::IndicatorMax)(u::AbstractArray{<:Any, 4}, return alpha end + +function create_cache(::Type{IndicatorClamp}, equations::AbstractEquations{2}, basis::LobattoLegendreBasis) + alpha = Vector{real(basis)}() + + weights = 0.5 * basis.weights + + return (; alpha, weights) +end + +function create_cache(typ::Type{IndicatorClamp}, mesh, equations::AbstractEquations{2}, dg::DGSEM, cache) + cache = create_cache(typ, equations, dg.basis) +end + +function (indicator_clamp::IndicatorClamp)(u::AbstractArray{<:Any,4}, + mesh, equations, dg::DGSEM, cache; + kwargs...) + @unpack alpha, weights = indicator_clamp.cache + resize!(alpha, nelements(dg, cache)) + + @threaded for element in eachelement(dg, cache) + mean = zero(real(dg.basis)) + + for j in eachnode(dg), i in eachnode(dg) + u_local = get_node_vars(u, equations, dg, i, j, element) + mean += indicator_clamp.variable(u_local, equations) * weights[i]*weights[j] + end + + if indicator_clamp.min <= mean <= indicator_clamp.max + alpha[element] = 1.0 + else + alpha[element] = -1.0 + end + end + + return alpha +end + end # @muladd From 81204099762a86b21280be828c9af48116f2d904 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Thu, 13 Jun 2024 18:36:12 +0200 Subject: [PATCH 44/81] Backup. --- src/auxiliary/t8code.jl | 6 +- src/meshes/mesh_io.jl | 70 ++++++++- src/meshes/t8code_mesh.jl | 299 +++++++++++++++++++++++++++++++++----- 3 files changed, 336 insertions(+), 39 deletions(-) diff --git a/src/auxiliary/t8code.jl b/src/auxiliary/t8code.jl index 7c1399fc803..c6bf45c128e 100644 --- a/src/auxiliary/t8code.jl +++ b/src/auxiliary/t8code.jl @@ -17,7 +17,8 @@ function init_t8code() # Initialize the sc library, has to happen before we initialize t8code. let catch_signals = 0, print_backtrace = 0, log_handler = C_NULL T8code.Libt8.sc_init(mpi_comm(), catch_signals, print_backtrace, log_handler, - T8code.Libt8.SC_LP_ERROR) + T8code.Libt8.SC_LP_DEBUG) + # T8code.Libt8.SC_LP_ERROR) end if T8code.Libt8.p4est_is_initialized() == 0 @@ -26,7 +27,8 @@ function init_t8code() end # Initialize t8code with log level ERROR to prevent a lot of output in AMR simulations. - t8_init(T8code.Libt8.SC_LP_ERROR) + # t8_init(T8code.Libt8.SC_LP_ERROR) + t8_init(T8code.Libt8.SC_LP_DEBUG) if haskey(ENV, "TRIXI_T8CODE_SC_FINALIZE") # Normally, `sc_finalize` should always be called during shutdown of an diff --git a/src/meshes/mesh_io.jl b/src/meshes/mesh_io.jl index b74a3b4d642..be14b155777 100644 --- a/src/meshes/mesh_io.jl +++ b/src/meshes/mesh_io.jl @@ -225,11 +225,50 @@ function save_mesh_file(mesh::P4estMesh, output_directory, timestep, mpi_paralle return filename end -# TODO: Implement this function as soon as there is support for this in `t8code`. -function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, mpi_parallel) - error("Mesh file output not supported yet for `T8codeMesh`.") +## # TODO: Implement this function as soon as there is support for this in `t8code`. +## function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, mpi_parallel) +## error("Mesh file output not supported yet for `T8codeMesh`.") +## +## return joinpath(output_directory, "dummy_mesh.h5") +## end + +function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, + mpi_parallel::False) + + elemIDs, neighIDs, faces, duals, orientations = get_cmesh_info(mesh) + + levels = trixi_t8_get_local_element_levels(mesh.forest) + + # Create output directory (if it does not exist). + mkpath(output_directory) + + # Determine file name based on existence of meaningful time step. + if timestep > 0 + filename = joinpath(output_directory, @sprintf("mesh_%06d.h5", timestep)) + else + filename = joinpath(output_directory, "mesh.h5") + end + + # Open file (clobber existing content). + h5open(filename, "w") do file + # Add context information as attributes. + attributes(file)["mesh_type"] = get_name(mesh) + attributes(file)["ndims"] = ndims(mesh) + attributes(file)["ntrees"] = t8_forest_get_num_local_trees(mesh.forest) + attributes(file)["nelements"] = ncells(mesh) - return joinpath(output_directory, "dummy_mesh.h5") + file["tree_node_coordinates"] = mesh.tree_node_coordinates + file["nodes"] = Vector(mesh.nodes) + file["boundary_names"] = mesh.boundary_names .|> String + file["elemIDs"] = elemIDs + file["neighIDs"] = neighIDs + file["faces"] = faces + file["duals"] = duals + file["orientations"] = orientations + file["levels"] = levels + end + + return filename end """ @@ -322,7 +361,29 @@ function load_mesh_serial(mesh_file::AbstractString; n_cells_max, RealT) mesh = P4estMesh{ndims}(p4est, tree_node_coordinates, nodes, boundary_names, mesh_file, false, true) + + elseif mesh_type == "T8codeMesh" + ndims, ntrees, nelements, tree_node_coordinates, + nodes, boundary_names_, elemIDs, neighIDs, faces, duals, orientations, levels = h5open(mesh_file, "r") do file + return read(attributes(file)["ndims"]), + read(attributes(file)["ntrees"]), + read(attributes(file)["nelements"]), + read(file["tree_node_coordinates"]), + read(file["nodes"]), + read(file["boundary_names"]), + read(file["elemIDs"]), + read(file["neighIDs"]), + read(file["faces"]), + read(file["duals"]), + read(file["orientations"]), + read(file["levels"]) + end + + boundary_names = boundary_names_ .|> Symbol + + mesh = T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, boundary_names, elemIDs, neighIDs, faces, duals, orientations, levels) else + error("Unknown mesh type!") end @@ -477,4 +538,5 @@ function load_mesh!(mesh::ParallelTreeMesh, mesh_file::AbstractString) return mesh end + end # @muladd diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 10a042be3ba..34eba17a535 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -108,6 +108,126 @@ Main mesh constructor for the `T8codeMesh` wrapping around a given t8code `forest` object. This constructor is typically called by other `T8codeMesh` constructors. +# Arguments +- `forest`: Pointer to a t8code forest. +- `boundary_names`: List of boundary names. +- `polydeg::Integer`: Polynomial degree used to store the geometry of the mesh. + The mapping will be approximated by an interpolation polynomial + of the specified degree for each tree. +- `mapping`: A function of `NDIMS` variables to describe the mapping that transforms + the imported mesh to the physical domain. Use `nothing` for the identity map. +""" + +function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, boundary_names, elemIDs, neighIDs, faces, duals, orientations, levels) + # Construct the cmesh from `interfaces` and `orientations`. + Trixi.cmesh_ref = Ref(t8_cmesh_t()) + t8_cmesh_init(Trixi.cmesh_ref) + cmesh = Trixi.cmesh_ref[] + + Trixi.linear_geom = Trixi.t8_geometry_linear_new(2) + Trixi.linear_geom_ptr = pointer_from_objref(Ref(Trixi.linear_geom)) + + # Use linear geometry for now. There is no real Lagrange geometry + # implementation yet in t8code. + t8_cmesh_register_geometry(cmesh, Trixi.linear_geom_ptr) + + N = length(nodes) + vertices = zeros(3 * 2^ndims) # quads/hexs only + + eclass = ndims > 2 ? T8_ECLASS_HEX : T8_ECLASS_QUAD + + for i = 1:ntrees + t8_cmesh_set_tree_class(cmesh, i-1, eclass) + + if ndims == 2 + vertices[1] = tree_node_coordinates[1,1,1,i] + vertices[2] = tree_node_coordinates[2,1,1,i] + + vertices[4] = tree_node_coordinates[1,N,1,i] + vertices[5] = tree_node_coordinates[2,N,1,i] + + vertices[7] = tree_node_coordinates[1,1,N,i] + vertices[8] = tree_node_coordinates[2,1,N,i] + + vertices[10] = tree_node_coordinates[1,N,N,i] + vertices[11] = tree_node_coordinates[2,N,N,i] + else + vertices[1] = tree_node_coordinates[1,1,1,1,i] + vertices[2] = tree_node_coordinates[2,1,1,1,i] + vertices[3] = tree_node_coordinates[3,1,1,1,i] + + vertices[4] = tree_node_coordinates[1,N,1,1,i] + vertices[5] = tree_node_coordinates[2,N,1,1,i] + vertices[6] = tree_node_coordinates[3,N,1,1,i] + + vertices[7] = tree_node_coordinates[1,1,N,1,i] + vertices[8] = tree_node_coordinates[2,1,N,1,i] + vertices[9] = tree_node_coordinates[3,1,N,1,i] + + vertices[10] = tree_node_coordinates[1,N,N,1,i] + vertices[11] = tree_node_coordinates[2,N,N,1,i] + vertices[12] = tree_node_coordinates[3,N,N,1,i] + + vertices[13] = tree_node_coordinates[1,1,1,N,i] + vertices[14] = tree_node_coordinates[2,1,1,N,i] + vertices[15] = tree_node_coordinates[3,1,1,N,i] + + vertices[16] = tree_node_coordinates[1,N,1,N,i] + vertices[17] = tree_node_coordinates[2,N,1,N,i] + vertices[18] = tree_node_coordinates[3,N,1,N,i] + + vertices[19] = tree_node_coordinates[1,1,N,N,i] + vertices[20] = tree_node_coordinates[2,1,N,N,i] + vertices[21] = tree_node_coordinates[3,1,N,N,i] + + vertices[22] = tree_node_coordinates[1,N,N,N,i] + vertices[23] = tree_node_coordinates[2,N,N,N,i] + vertices[24] = tree_node_coordinates[3,N,N,N,i] + end + + t8_cmesh_set_tree_vertices(cmesh, i-1, vertices, 3) + end + + for i = 1:length(elemIDs) + t8_cmesh_set_join(cmesh, elemIDs[i], neighIDs[i], faces[i], duals[i], orientations[i]) + end + + t8_cmesh_commit(cmesh, mpi_comm()) + + do_face_ghost = mpi_isparallel() + scheme = t8_scheme_new_default_cxx() + initial_refinement_level = 0 + forest = t8_forest_new_uniform(cmesh, scheme, initial_refinement_level, do_face_ghost, mpi_comm()) + + function adapt_callback(forest, ltreeid, eclass_scheme, lelementid, elements, is_family, + user_data) + + level = t8_element_level(eclass_scheme, elements[1]) + + if level < levels[lelementid + 1] + return 1 + elseif level > levels[lelementid + 1] + return -1 + end + + return 0 + end + + for level = 0:maximum(levels) + forest = adapt(forest, adapt_callback; recursive = false, balance = false, + partition = false, ghost = false, user_data = C_NULL) + end + + return T8codeMesh{ndims}(forest, tree_node_coordinates, nodes, boundary_names, "") +end + +""" + T8codeMesh{NDIMS, RealT}(forest, boundary_names; polydeg = 1, mapping = nothing) + +Main mesh constructor for the `T8codeMesh` wrapping around a given t8code +`forest` object. This constructor is typically called by other `T8codeMesh` +constructors. + # Arguments - `forest`: Pointer to a t8code forest. - `boundary_names`: List of boundary names. @@ -702,10 +822,10 @@ Adapt a `T8codeMesh` according to a user-defined `adapt_callback`. - `ghost = true`: Create a ghost layer for MPI data exchange. - `user_data = C_NULL`: Pointer to some arbitrary user-defined data. """ -function adapt!(mesh::T8codeMesh, adapt_callback; recursive = true, balance = true, +function adapt(forest::Ptr{t8_forest}, adapt_callback; recursive = true, balance = true, partition = true, ghost = true, user_data = C_NULL) # Check that forest is a committed, that is valid and usable, forest. - @assert t8_forest_is_committed(mesh.forest) != 0 + @assert t8_forest_is_committed(forest) != 0 # Init new forest. new_forest_ref = Ref{t8_forest_t}() @@ -718,7 +838,7 @@ function adapt!(mesh::T8codeMesh, adapt_callback; recursive = true, balance = tr t8_forest_set_user_data(new_forest, pointer_from_objref(Ref(adapt_callback_passthrough(adapt_callback, user_data)))) - t8_forest_set_adapt(new_forest, mesh.forest, + t8_forest_set_adapt(new_forest, forest, @t8_adapt_callback(adapt_callback_wrapper), recursive) if balance @@ -736,9 +856,12 @@ function adapt!(mesh::T8codeMesh, adapt_callback; recursive = true, balance = tr t8_forest_commit(new_forest) end - mesh.forest = new_forest + return new_forest +end - return nothing +function adapt!(mesh::T8codeMesh, adapt_callback; kwargs...) + # Call `t8_forest_ref(Ref(mesh.forest))` to keep it. + mesh.forest = adapt(mesh.forest, adapt_callback; kwargs...) end """ @@ -796,10 +919,14 @@ function get_global_first_element_ids(mesh::T8codeMesh) end function count_interfaces(mesh::T8codeMesh) - @assert t8_forest_is_committed(mesh.forest) != 0 + return count_interfaces(mesh.forest,ndims(mesh)) +end - num_local_elements = t8_forest_get_local_num_elements(mesh.forest) - num_local_trees = t8_forest_get_num_local_trees(mesh.forest) +function count_interfaces(forest::Ptr{t8_forest}, ndims) + @assert t8_forest_is_committed(forest) != 0 + + num_local_elements = t8_forest_get_local_num_elements(forest) + num_local_trees = t8_forest_get_num_local_trees(forest) current_index = t8_locidx_t(0) @@ -812,30 +939,30 @@ function count_interfaces(mesh::T8codeMesh) visited_global_mortar_ids = Set{UInt64}([]) - max_level = t8_forest_get_maxlevel(mesh.forest) #UInt64 - max_tree_num_elements = UInt64(2^ndims(mesh))^max_level + max_level = t8_forest_get_maxlevel(forest) #UInt64 + max_tree_num_elements = UInt64(2^ndims)^max_level if mpi_isparallel() - ghost_num_trees = t8_forest_ghost_num_trees(mesh.forest) + ghost_num_trees = t8_forest_ghost_num_trees(forest) ghost_tree_element_offsets = [num_local_elements + - t8_forest_ghost_get_tree_element_offset(mesh.forest, + t8_forest_ghost_get_tree_element_offset(forest, itree) for itree in 0:(ghost_num_trees - 1)] - ghost_global_treeids = [t8_forest_ghost_get_global_treeid(mesh.forest, itree) + ghost_global_treeids = [t8_forest_ghost_get_global_treeid(forest, itree) for itree in 0:(ghost_num_trees - 1)] end for itree in 0:(num_local_trees - 1) - tree_class = t8_forest_get_tree_class(mesh.forest, itree) - eclass_scheme = t8_forest_get_eclass_scheme(mesh.forest, tree_class) + tree_class = t8_forest_get_tree_class(forest, itree) + eclass_scheme = t8_forest_get_eclass_scheme(forest, tree_class) - num_elements_in_tree = t8_forest_get_tree_num_elements(mesh.forest, itree) + num_elements_in_tree = t8_forest_get_tree_num_elements(forest, itree) - global_itree = t8_forest_global_tree_id(mesh.forest, itree) + global_itree = t8_forest_global_tree_id(forest, itree) for ielement in 0:(num_elements_in_tree - 1) - element = t8_forest_get_element_in_tree(mesh.forest, itree, ielement) + element = t8_forest_get_element_in_tree(forest, itree, ielement) level = t8_element_level(eclass_scheme, element) @@ -855,7 +982,7 @@ function count_interfaces(mesh::T8codeMesh) forest_is_balanced = Cint(1) - t8_forest_leaf_face_neighbors(mesh.forest, itree, element, + t8_forest_leaf_face_neighbors(forest, itree, element, pneighbor_leaves_ref, iface, dual_faces_ref, num_neighbors_ref, pelement_indices_ref, pneigh_scheme_ref, @@ -989,6 +1116,8 @@ function fill_mesh_info!(mesh::T8codeMesh, interfaces, mortars, boundaries, visited_global_mortar_ids = Set{UInt64}([]) global_mortar_id_to_local = Dict{UInt64, Int}([]) + cmesh = t8_forest_get_cmesh(mesh.forest) + # Loop over all local trees. for itree in 0:(num_local_trees - 1) tree_class = t8_forest_get_tree_class(mesh.forest, itree) @@ -1012,20 +1141,6 @@ function fill_mesh_info!(mesh::T8codeMesh, interfaces, mortars, boundaries, # Loop over all faces of the current local element. for iface in 0:(num_faces - 1) - # Compute the `orientation` of the touching faces. - if t8_element_is_root_boundary(eclass_scheme, element, iface) == 1 - cmesh = t8_forest_get_cmesh(mesh.forest) - itree_in_cmesh = t8_forest_ltreeid_to_cmesh_ltreeid(mesh.forest, itree) - iface_in_tree = t8_element_tree_face(eclass_scheme, element, iface) - orientation_ref = Ref{Cint}() - - t8_cmesh_get_face_neighbor(cmesh, itree_in_cmesh, iface_in_tree, C_NULL, - orientation_ref) - orientation = orientation_ref[] - else - orientation = zero(Cint) - end - pelement_indices_ref = Ref{Ptr{t8_locidx_t}}() pneighbor_leaves_ref = Ref{Ptr{Ptr{t8_element}}}() pneigh_scheme_ref = Ref{Ptr{t8_eclass_scheme}}() @@ -1092,6 +1207,19 @@ function fill_mesh_info!(mesh::T8codeMesh, interfaces, mortars, boundaries, else neighbor_level = t8_element_level(neighbor_scheme, neighbor_leaves[1]) + # Compute the `orientation` of the touching faces. + if t8_element_is_root_boundary(eclass_scheme, element, iface) == 1 + itree_in_cmesh = t8_forest_ltreeid_to_cmesh_ltreeid(mesh.forest, itree) + iface_in_tree = t8_element_tree_face(eclass_scheme, element, iface) + orientation_ref = Ref{Cint}() + + t8_cmesh_get_face_neighbor(cmesh, itree_in_cmesh, iface_in_tree, C_NULL, + orientation_ref) + orientation = orientation_ref[] + else + orientation = zero(Cint) + end + # Local interface or mortar. if all(neighbor_ielements .< num_local_elements) @@ -1263,6 +1391,111 @@ function fill_mesh_info!(mesh::T8codeMesh, interfaces, mortars, boundaries, return nothing end +function get_cmesh_info(mesh::T8codeMesh) + @assert t8_forest_is_committed(mesh.forest) != 0 + cmesh = t8_forest_get_cmesh(mesh.forest) + return get_cmesh_info(cmesh, ndims(mesh)) +end + +function get_cmesh_info(cmesh::Ptr{t8_cmesh}, ndims) + # Avoid destroying cmesh when destroying the forest. + t8_cmesh_ref(cmesh) + + # In order to retrieve the connectivity between cmesh elements, a uniform + # forest with one element per cmesh cell is initialized. + scheme = t8_scheme_new_default_cxx() + do_face_ghost = 0 + initial_refinement_level = 0 + forest = t8_forest_new_uniform(cmesh, scheme, initial_refinement_level, do_face_ghost, mpi_comm()) + + counts = count_interfaces(forest, ndims) + + elemIDs = zeros(Int, counts.interfaces) + neighIDs = zeros(Int, counts.interfaces) + orientations = zeros(Int32, counts.interfaces) + faces = zeros(Int8, counts.interfaces) + duals = zeros(Int8, counts.interfaces) + + num_local_elements = t8_forest_get_local_num_elements(forest) + num_local_trees = t8_forest_get_num_local_trees(forest) + + sfc_index = 0 # space-filling curve index + itf_index = 0 # interface index + + # Loop over all local trees. + for itree in 0:(num_local_trees - 1) + tree_class = t8_forest_get_tree_class(forest, itree) + eclass_scheme = t8_forest_get_eclass_scheme(forest, tree_class) + + num_elements_in_tree = t8_forest_get_tree_num_elements(forest, itree) + + # Loop over all local elements of the current local tree. + for ielement in 0:(num_elements_in_tree - 1) + element = t8_forest_get_element_in_tree(forest, itree, ielement) + level = t8_element_level(eclass_scheme, element) + num_faces = t8_element_num_faces(eclass_scheme, element) + + # Loop over all faces of the current local element. + for iface in 0:(num_faces - 1) + # Compute the `orientation` of the touching faces. + if t8_element_is_root_boundary(eclass_scheme, element, iface) == 1 + itree_in_cmesh = t8_forest_ltreeid_to_cmesh_ltreeid(forest, itree) + iface_in_tree = t8_element_tree_face(eclass_scheme, element, iface) + orientation_ref = Ref{Cint}() + + t8_cmesh_get_face_neighbor(cmesh, itree_in_cmesh, iface_in_tree, C_NULL, + orientation_ref) + orientation = orientation_ref[] + else + orientation = zero(Cint) + end + + pelement_indices_ref = Ref{Ptr{t8_locidx_t}}() + pneighbor_leaves_ref = Ref{Ptr{Ptr{t8_element}}}() + pneigh_scheme_ref = Ref{Ptr{t8_eclass_scheme}}() + + dual_faces_ref = Ref{Ptr{Cint}}() + num_neighbors_ref = Ref{Cint}() + + forest_is_balanced = Cint(1) + + # Query neighbor information from t8code. + t8_forest_leaf_face_neighbors(forest, itree, element, + pneighbor_leaves_ref, iface, dual_faces_ref, + num_neighbors_ref, + pelement_indices_ref, pneigh_scheme_ref, + forest_is_balanced) + + num_neighbors = num_neighbors_ref[] + dual_faces = unsafe_wrap(Array, dual_faces_ref[], num_neighbors) + neighbor_ielements = unsafe_wrap(Array, pelement_indices_ref[], num_neighbors) + neighbor_leaves = unsafe_wrap(Array, pneighbor_leaves_ref[], num_neighbors) + neighbor_scheme = pneigh_scheme_ref[] + + # The condition ensures we only visit the interface once. + if num_neighbors == 1 && sfc_index <= neighbor_ielements[1] + itf_index += 1 + elemIDs[itf_index] = sfc_index + neighIDs[itf_index] = neighbor_ielements[1] + orientations[itf_index] = orientation + faces[itf_index] = iface + duals[itf_index] = dual_faces[1] + end + + t8_free(dual_faces_ref[]) + t8_free(pneighbor_leaves_ref[]) + t8_free(pelement_indices_ref[]) + end # for iface + + sfc_index += 1 + end # for ielement + end # for itree + + t8_forest_unref(Ref(forest)) + + return elemIDs, neighIDs, faces, duals, orientations +end + #! format: off @deprecate T8codeMesh{2}(conn::Ptr{p4est_connectivity}; kwargs...) T8codeMesh(conn::Ptr{p4est_connectivity}; kwargs...) @deprecate T8codeMesh{3}(conn::Ptr{p8est_connectivity}; kwargs...) T8codeMesh(conn::Ptr{p8est_connectivity}; kwargs...) From 3ad35154c4b0ce67be822bcb5af497223db79630 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Fri, 14 Jun 2024 19:07:49 +0200 Subject: [PATCH 45/81] Add save callback to elixir. --- .../t8code_2d_dgsem/elixir_advection_unstructured_flag.jl | 6 +++++- src/meshes/t8code_mesh.jl | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl b/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl index e512f328234..d966ef2273d 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl @@ -53,13 +53,17 @@ summary_callback = SummaryCallback() # prints the results. analysis_callback = AnalysisCallback(semi, interval = 100) +# The SaveSolutionCallback allows to save the solution to a file in regular intervals +save_solution = SaveSolutionCallback(interval = 100, + solution_variables = cons2prim) + # The StepsizeCallback handles the re-calculation of the maximum Δt after each # time step. stepsize_callback = StepsizeCallback(cfl = 1.4) # Create a CallbackSet to collect all callbacks such that they can be passed to # the ODE solver. -callbacks = CallbackSet(summary_callback, analysis_callback, stepsize_callback) +callbacks = CallbackSet(summary_callback, analysis_callback, save_solution, stepsize_callback) ############################################################################### # Run the simulation. diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 34eba17a535..f30916c104b 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -26,6 +26,8 @@ mutable struct T8codeMesh{NDIMS, RealT <: Real, IsParallel, NDIMSP2, NNODES} <: nmpiinterfaces :: Int nmpimortars :: Int + unsaved_changes :: Bool + function T8codeMesh{NDIMS}(forest::Ptr{t8_forest}, tree_node_coordinates, nodes, boundary_names, current_filename) where {NDIMS} @@ -38,6 +40,7 @@ mutable struct T8codeMesh{NDIMS, RealT <: Real, IsParallel, NDIMSP2, NNODES} <: mesh.boundary_names = boundary_names mesh.current_filename = current_filename mesh.tree_node_coordinates = tree_node_coordinates + mesh.unsaved_changes = true finalizer(mesh) do mesh # When finalizing `mesh.forest`, `mesh.scheme` and `mesh.cmesh` are From 23714f1eaa303327770a59f17a616a100e9e7451 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Tue, 18 Jun 2024 16:09:23 +0200 Subject: [PATCH 46/81] Backup. --- src/auxiliary/t8code.jl | 4 +- src/meshes/mesh_io.jl | 81 +++++++++++++++++++++++++++++++-------- src/meshes/t8code_mesh.jl | 34 ++++++++++------ 3 files changed, 89 insertions(+), 30 deletions(-) diff --git a/src/auxiliary/t8code.jl b/src/auxiliary/t8code.jl index c6bf45c128e..9017d8fb851 100644 --- a/src/auxiliary/t8code.jl +++ b/src/auxiliary/t8code.jl @@ -52,7 +52,7 @@ function trixi_t8_get_local_element_levels(forest) # Check that forest is a committed, that is valid and usable, forest. @assert t8_forest_is_committed(forest) != 0 - levels = Vector{Int}(undef, t8_forest_get_local_num_elements(forest)) + levels = Vector{UInt8}(undef, t8_forest_get_local_num_elements(forest)) # Get the number of trees that have elements of this process. num_local_trees = t8_forest_get_num_local_trees(forest) @@ -69,7 +69,7 @@ function trixi_t8_get_local_element_levels(forest) for ielement in 0:(num_elements_in_tree - 1) element = t8_forest_get_element_in_tree(forest, itree, ielement) current_index += 1 - levels[current_index] = t8_element_level(eclass_scheme, element) + levels[current_index] = UInt8(t8_element_level(eclass_scheme, element)) end # for end # for diff --git a/src/meshes/mesh_io.jl b/src/meshes/mesh_io.jl index be14b155777..c562853bb7f 100644 --- a/src/meshes/mesh_io.jl +++ b/src/meshes/mesh_io.jl @@ -225,22 +225,11 @@ function save_mesh_file(mesh::P4estMesh, output_directory, timestep, mpi_paralle return filename end -## # TODO: Implement this function as soon as there is support for this in `t8code`. -## function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, mpi_parallel) -## error("Mesh file output not supported yet for `T8codeMesh`.") -## -## return joinpath(output_directory, "dummy_mesh.h5") -## end - function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, - mpi_parallel::False) - - elemIDs, neighIDs, faces, duals, orientations = get_cmesh_info(mesh) - - levels = trixi_t8_get_local_element_levels(mesh.forest) + mpi_parallel) # Create output directory (if it does not exist). - mkpath(output_directory) + mpi_isroot() && mkpath(output_directory) # Determine file name based on existence of meaningful time step. if timestep > 0 @@ -249,12 +238,38 @@ function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, filename = joinpath(output_directory, "mesh.h5") end + levels = get_levels(mesh) + if mpi_parallel + levels = MPI.Gather(levels, mpi_root(), mpi_comm()) + end + + num_global_trees = t8_forest_get_num_global_trees(mesh.forest) + num_elements_per_tree = zeros(t8_gloidx_t, num_global_trees) + num_local_trees = t8_forest_get_num_local_trees(mesh.forest) + for local_tree_id in 0:(num_local_trees - 1) + num_local_elements_in_tree = t8_forest_get_tree_num_elements(mesh.forest, local_tree_id) + global_tree_id = t8_forest_global_tree_id(mesh.forest, local_tree_id) + num_elements_per_tree[global_tree_id + 1] = num_local_elements_in_tree + end + + if mpi_parallel + num_elements_per_tree = MPI.Reduce!(num_elements_per_tree, +, mpi_comm()) + end + + # Since the mesh attributes are replicated on all ranks, only save from MPI + # root. + if !mpi_isroot() + return filename + end + + elemIDs, neighIDs, faces, duals, orientations = get_cmesh_info(mesh) + # Open file (clobber existing content). h5open(filename, "w") do file # Add context information as attributes. attributes(file)["mesh_type"] = get_name(mesh) attributes(file)["ndims"] = ndims(mesh) - attributes(file)["ntrees"] = t8_forest_get_num_local_trees(mesh.forest) + attributes(file)["ntrees"] = ntrees(mesh) attributes(file)["nelements"] = ncells(mesh) file["tree_node_coordinates"] = mesh.tree_node_coordinates @@ -266,6 +281,7 @@ function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, file["duals"] = duals file["orientations"] = orientations file["levels"] = levels + file["num_elements_per_tree"] = num_elements_per_tree end return filename @@ -364,7 +380,7 @@ function load_mesh_serial(mesh_file::AbstractString; n_cells_max, RealT) elseif mesh_type == "T8codeMesh" ndims, ntrees, nelements, tree_node_coordinates, - nodes, boundary_names_, elemIDs, neighIDs, faces, duals, orientations, levels = h5open(mesh_file, "r") do file + nodes, boundary_names_, elemIDs, neighIDs, faces, duals, orientations, levels, num_elements_per_tree = h5open(mesh_file, "r") do file return read(attributes(file)["ndims"]), read(attributes(file)["ntrees"]), read(attributes(file)["nelements"]), @@ -376,12 +392,13 @@ function load_mesh_serial(mesh_file::AbstractString; n_cells_max, RealT) read(file["faces"]), read(file["duals"]), read(file["orientations"]), - read(file["levels"]) + read(file["levels"]), + read(file["num_elements_per_tree"]) end boundary_names = boundary_names_ .|> Symbol - mesh = T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, boundary_names, elemIDs, neighIDs, faces, duals, orientations, levels) + mesh = T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, boundary_names, elemIDs, neighIDs, faces, duals, orientations, levels, num_elements_per_tree) else error("Unknown mesh type!") @@ -472,6 +489,36 @@ function load_mesh_parallel(mesh_file::AbstractString; n_cells_max, RealT) mesh = P4estMesh{ndims_}(p4est, tree_node_coordinates, nodes, boundary_names, mesh_file, false, true) + + elseif mesh_type == "T8codeMesh" + + if mpi_isroot() + ndims, ntrees, nelements, tree_node_coordinates, + nodes, boundary_names_, elemIDs, neighIDs, faces, duals, orientations, levels = h5open(mesh_file, "r") do file + return read(attributes(file)["ndims"]), + read(attributes(file)["ntrees"]), + read(attributes(file)["nelements"]), + read(file["tree_node_coordinates"]), + read(file["nodes"]), + read(file["boundary_names"]), + read(file["elemIDs"]), + read(file["neighIDs"]), + read(file["faces"]), + read(file["duals"]), + read(file["orientations"]), + read(file["levels"]) + end + + boundary_names = boundary_names_ .|> Symbol + + data = (ndims, ntrees, nelements, tree_node_coordinates, nodes, boundary_names, elemIDs, neighIDs, faces, duals, orientations, levels) + MPI.bcast(data, mpi_root(), mpi_comm()) + else + data = MPI.bcast(nothing, mpi_root(), mpi_comm()) + ndims, ntrees, nelements, tree_node_coordinates, nodes, boundary_names, elemIDs, neighIDs, faces, duals, orientations, levels = data + end + + mesh = T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, boundary_names, elemIDs, neighIDs, faces, duals, orientations, levels) else error("Unknown mesh type!") end diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index f30916c104b..7a0a11597ba 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -120,8 +120,7 @@ constructors. - `mapping`: A function of `NDIMS` variables to describe the mapping that transforms the imported mesh to the physical domain. Use `nothing` for the identity map. """ - -function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, boundary_names, elemIDs, neighIDs, faces, duals, orientations, levels) +function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, boundary_names, elemIDs, neighIDs, faces, duals, orientations, levels, num_elements_per_tree) # Construct the cmesh from `interfaces` and `orientations`. Trixi.cmesh_ref = Ref(t8_cmesh_t()) t8_cmesh_init(Trixi.cmesh_ref) @@ -201,25 +200,34 @@ function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, boun scheme = t8_scheme_new_default_cxx() initial_refinement_level = 0 forest = t8_forest_new_uniform(cmesh, scheme, initial_refinement_level, do_face_ghost, mpi_comm()) - - function adapt_callback(forest, ltreeid, eclass_scheme, lelementid, elements, is_family, + + virtual_element_index = 1 + + cum_sum_num_elements_per_tree = cumsum(num_elements_per_tree) + + function adapt_callback(forest, local_tree_id, eclass_scheme, local_element_id, elements, is_family, user_data) + global_tree_id = t8_forest_global_tree_id(forest, local_tree_id) + + if virtual_element_index > cum_sum_num_elements_per_tree[global_tree_id + 1] + return 0 + end + level = t8_element_level(eclass_scheme, elements[1]) - if level < levels[lelementid + 1] + if level < levels[virtual_element_index] return 1 - elseif level > levels[lelementid + 1] - return -1 end + virtual_element_index += 1 return 0 end - for level = 0:maximum(levels) - forest = adapt(forest, adapt_callback; recursive = false, balance = false, - partition = false, ghost = false, user_data = C_NULL) - end + forest = adapt(forest, adapt_callback; recursive = true, balance = false, + partition = false, ghost = false, user_data = C_NULL) + + # forest = t8_forest_partition(forest) return T8codeMesh{ndims}(forest, tree_node_coordinates, nodes, boundary_names, "") end @@ -1394,6 +1402,10 @@ function fill_mesh_info!(mesh::T8codeMesh, interfaces, mortars, boundaries, return nothing end +function get_levels(mesh::T8codeMesh) + return trixi_t8_get_local_element_levels(mesh.forest) +end + function get_cmesh_info(mesh::T8codeMesh) @assert t8_forest_is_committed(mesh.forest) != 0 cmesh = t8_forest_get_cmesh(mesh.forest) From 3f8ded29140d742c84cc70d05c50b48495e07442 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Wed, 19 Jun 2024 17:40:09 +0200 Subject: [PATCH 47/81] Refined code. Make it work in parallel. --- .../elixir_advection_unstructured_curved.jl | 6 +- src/meshes/mesh_io.jl | 46 +++-- src/meshes/t8code_mesh.jl | 166 +++++++----------- 3 files changed, 98 insertions(+), 120 deletions(-) diff --git a/examples/t8code_3d_dgsem/elixir_advection_unstructured_curved.jl b/examples/t8code_3d_dgsem/elixir_advection_unstructured_curved.jl index fe6aa48e7d9..df1dbce2af1 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_unstructured_curved.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_unstructured_curved.jl @@ -69,6 +69,10 @@ summary_callback = SummaryCallback() analysis_interval = 100 analysis_callback = AnalysisCallback(semi, interval = analysis_interval) +# The SaveSolutionCallback allows to save the solution to a file in regular intervals +save_solution = SaveSolutionCallback(interval = 100, + solution_variables = cons2prim) + alive_callback = AliveCallback(analysis_interval = analysis_interval) # The StepsizeCallback handles the re-calculation of the maximum Δt after each time step @@ -76,7 +80,7 @@ stepsize_callback = StepsizeCallback(cfl = 1.2) # Create a CallbackSet to collect all callbacks such that they can be passed to the ODE solver callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, - stepsize_callback) + save_solution, stepsize_callback) ############################################################################### # run the simulation diff --git a/src/meshes/mesh_io.jl b/src/meshes/mesh_io.jl index c562853bb7f..31c73ef6537 100644 --- a/src/meshes/mesh_io.jl +++ b/src/meshes/mesh_io.jl @@ -226,7 +226,7 @@ function save_mesh_file(mesh::P4estMesh, output_directory, timestep, mpi_paralle end function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, - mpi_parallel) + mpi_parallel::Any) # Create output directory (if it does not exist). mpi_isroot() && mkpath(output_directory) @@ -238,11 +238,13 @@ function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, filename = joinpath(output_directory, "mesh.h5") end + # Retrieve refinement levels of all elements. levels = get_levels(mesh) - if mpi_parallel + if mpi_isparallel() levels = MPI.Gather(levels, mpi_root(), mpi_comm()) end + # Retrieve number of elements per tree. num_global_trees = t8_forest_get_num_global_trees(mesh.forest) num_elements_per_tree = zeros(t8_gloidx_t, num_global_trees) num_local_trees = t8_forest_get_num_local_trees(mesh.forest) @@ -252,7 +254,7 @@ function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, num_elements_per_tree[global_tree_id + 1] = num_local_elements_in_tree end - if mpi_parallel + if mpi_isparallel() num_elements_per_tree = MPI.Reduce!(num_elements_per_tree, +, mpi_comm()) end @@ -262,7 +264,8 @@ function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, return filename end - elemIDs, neighIDs, faces, duals, orientations = get_cmesh_info(mesh) + # Retrieve face connectivity info of the coarse mesh. + treeIDs, neighIDs, faces, duals, orientations = get_cmesh_info(mesh) # Open file (clobber existing content). h5open(filename, "w") do file @@ -275,7 +278,7 @@ function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, file["tree_node_coordinates"] = mesh.tree_node_coordinates file["nodes"] = Vector(mesh.nodes) file["boundary_names"] = mesh.boundary_names .|> String - file["elemIDs"] = elemIDs + file["treeIDs"] = treeIDs file["neighIDs"] = neighIDs file["faces"] = faces file["duals"] = duals @@ -380,14 +383,15 @@ function load_mesh_serial(mesh_file::AbstractString; n_cells_max, RealT) elseif mesh_type == "T8codeMesh" ndims, ntrees, nelements, tree_node_coordinates, - nodes, boundary_names_, elemIDs, neighIDs, faces, duals, orientations, levels, num_elements_per_tree = h5open(mesh_file, "r") do file + nodes, boundary_names_, treeIDs, neighIDs, faces, duals, orientations, + levels, num_elements_per_tree = h5open(mesh_file, "r") do file return read(attributes(file)["ndims"]), read(attributes(file)["ntrees"]), read(attributes(file)["nelements"]), read(file["tree_node_coordinates"]), read(file["nodes"]), read(file["boundary_names"]), - read(file["elemIDs"]), + read(file["treeIDs"]), read(file["neighIDs"]), read(file["faces"]), read(file["duals"]), @@ -398,7 +402,9 @@ function load_mesh_serial(mesh_file::AbstractString; n_cells_max, RealT) boundary_names = boundary_names_ .|> Symbol - mesh = T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, boundary_names, elemIDs, neighIDs, faces, duals, orientations, levels, num_elements_per_tree) + mesh = T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, + nodes, boundary_names, treeIDs, neighIDs, faces, + duals, orientations, levels, num_elements_per_tree) else error("Unknown mesh type!") @@ -489,36 +495,42 @@ function load_mesh_parallel(mesh_file::AbstractString; n_cells_max, RealT) mesh = P4estMesh{ndims_}(p4est, tree_node_coordinates, nodes, boundary_names, mesh_file, false, true) - elseif mesh_type == "T8codeMesh" - if mpi_isroot() - ndims, ntrees, nelements, tree_node_coordinates, - nodes, boundary_names_, elemIDs, neighIDs, faces, duals, orientations, levels = h5open(mesh_file, "r") do file + ndims, ntrees, nelements, tree_node_coordinates, nodes, + boundary_names_, treeIDs, neighIDs, faces, duals, orientations, levels, + num_elements_per_tree = h5open(mesh_file, "r") do file return read(attributes(file)["ndims"]), read(attributes(file)["ntrees"]), read(attributes(file)["nelements"]), read(file["tree_node_coordinates"]), read(file["nodes"]), read(file["boundary_names"]), - read(file["elemIDs"]), + read(file["treeIDs"]), read(file["neighIDs"]), read(file["faces"]), read(file["duals"]), read(file["orientations"]), - read(file["levels"]) + read(file["levels"]), + read(file["num_elements_per_tree"]) end boundary_names = boundary_names_ .|> Symbol - data = (ndims, ntrees, nelements, tree_node_coordinates, nodes, boundary_names, elemIDs, neighIDs, faces, duals, orientations, levels) + data = (ndims, ntrees, nelements, tree_node_coordinates, nodes, + boundary_names, treeIDs, neighIDs, faces, duals, + orientations, levels, num_elements_per_tree) MPI.bcast(data, mpi_root(), mpi_comm()) else data = MPI.bcast(nothing, mpi_root(), mpi_comm()) - ndims, ntrees, nelements, tree_node_coordinates, nodes, boundary_names, elemIDs, neighIDs, faces, duals, orientations, levels = data + ndims, ntrees, nelements, tree_node_coordinates, nodes, + boundary_names, treeIDs, neighIDs, faces, duals, orientations, levels, + num_elements_per_tree = data end - mesh = T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, boundary_names, elemIDs, neighIDs, faces, duals, orientations, levels) + mesh = T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, + nodes, boundary_names, treeIDs, neighIDs, faces, + duals, orientations, levels, num_elements_per_tree) else error("Unknown mesh type!") end diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 7a0a11597ba..9fe6836db54 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -120,24 +120,25 @@ constructors. - `mapping`: A function of `NDIMS` variables to describe the mapping that transforms the imported mesh to the physical domain. Use `nothing` for the identity map. """ -function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, boundary_names, elemIDs, neighIDs, faces, duals, orientations, levels, num_elements_per_tree) - # Construct the cmesh from `interfaces` and `orientations`. +function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, + boundary_names, elemIDs, neighIDs, faces, duals, + orientations, levels, num_elements_per_tree) Trixi.cmesh_ref = Ref(t8_cmesh_t()) t8_cmesh_init(Trixi.cmesh_ref) cmesh = Trixi.cmesh_ref[] - Trixi.linear_geom = Trixi.t8_geometry_linear_new(2) - Trixi.linear_geom_ptr = pointer_from_objref(Ref(Trixi.linear_geom)) - # Use linear geometry for now. There is no real Lagrange geometry # implementation yet in t8code. + Trixi.linear_geom = Trixi.t8_geometry_linear_new(2) + Trixi.linear_geom_ptr = pointer_from_objref(Ref(Trixi.linear_geom)) t8_cmesh_register_geometry(cmesh, Trixi.linear_geom_ptr) - N = length(nodes) - vertices = zeros(3 * 2^ndims) # quads/hexs only - + # Determine element class. eclass = ndims > 2 ? T8_ECLASS_HEX : T8_ECLASS_QUAD + # Store element vertices inside the cmesh. + N = length(nodes) + vertices = zeros(3 * 2^ndims) # quads/hexs only for i = 1:ntrees t8_cmesh_set_tree_class(cmesh, i-1, eclass) @@ -190,44 +191,58 @@ function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, boun t8_cmesh_set_tree_vertices(cmesh, i-1, vertices, 3) end + # Connect the coarse mesh elements. for i = 1:length(elemIDs) t8_cmesh_set_join(cmesh, elemIDs[i], neighIDs[i], faces[i], duals[i], orientations[i]) end t8_cmesh_commit(cmesh, mpi_comm()) + # Init a new forest with just one element per tree. do_face_ghost = mpi_isparallel() scheme = t8_scheme_new_default_cxx() initial_refinement_level = 0 forest = t8_forest_new_uniform(cmesh, scheme, initial_refinement_level, do_face_ghost, mpi_comm()) - virtual_element_index = 1 - cum_sum_num_elements_per_tree = cumsum(num_elements_per_tree) + # Compute the offset within the to-be-reconstructed forest depending on the + # MPI rank resp. first global tree id. + virtual_element_index = 1 # one-based index + if mpi_rank() > 0 + last_global_tree_id_of_preceding_rank = t8_forest_global_tree_id(forest, 0) - 1 + virtual_element_index += cum_sum_num_elements_per_tree[last_global_tree_id_of_preceding_rank + 1] + end + function adapt_callback(forest, local_tree_id, eclass_scheme, local_element_id, elements, is_family, user_data) + # Check if we are already in the next tree in terms of the `virtual_element_index`. global_tree_id = t8_forest_global_tree_id(forest, local_tree_id) - if virtual_element_index > cum_sum_num_elements_per_tree[global_tree_id + 1] return 0 end + # Test if we already reached the targeted level. level = t8_element_level(eclass_scheme, elements[1]) - if level < levels[virtual_element_index] - return 1 + return 1 # Go one refinement level deeper. end + # Targeted level is reached. virtual_element_index += 1 return 0 end + # The adapt callback refines the forest according to the `levels` array. + # For each tree the callback recursively increases the refinement level + # till it matches with the associated section in `levels. forest = adapt(forest, adapt_callback; recursive = true, balance = false, partition = false, ghost = false, user_data = C_NULL) - # forest = t8_forest_partition(forest) + # if mpi_isparallel() + # forest = t8_forest_partition(forest) + # end return T8codeMesh{ndims}(forest, tree_node_coordinates, nodes, boundary_names, "") end @@ -1413,102 +1428,49 @@ function get_cmesh_info(mesh::T8codeMesh) end function get_cmesh_info(cmesh::Ptr{t8_cmesh}, ndims) - # Avoid destroying cmesh when destroying the forest. - t8_cmesh_ref(cmesh) + num_trees = t8_cmesh_get_num_trees(cmesh) + num_faces = 2*ndims - # In order to retrieve the connectivity between cmesh elements, a uniform - # forest with one element per cmesh cell is initialized. - scheme = t8_scheme_new_default_cxx() - do_face_ghost = 0 - initial_refinement_level = 0 - forest = t8_forest_new_uniform(cmesh, scheme, initial_refinement_level, do_face_ghost, mpi_comm()) + num_interfaces = 0 - counts = count_interfaces(forest, ndims) + dual_face_ref = Ref{Cint}() + orientation_ref = Ref{Cint}() - elemIDs = zeros(Int, counts.interfaces) - neighIDs = zeros(Int, counts.interfaces) - orientations = zeros(Int32, counts.interfaces) - faces = zeros(Int8, counts.interfaces) - duals = zeros(Int8, counts.interfaces) + # Count connected faces. + for itree in 0:(num_trees - 1) + for iface in 0:(num_faces - 1) + neigh_itree = t8_cmesh_get_face_neighbor(cmesh, itree, iface, dual_face_ref, C_NULL) + if itree < neigh_itree || itree == neigh_itree && iface < dual_face_ref[] + num_interfaces += 1 + end + end + end - num_local_elements = t8_forest_get_local_num_elements(forest) - num_local_trees = t8_forest_get_num_local_trees(forest) + # Allocate arrays. + treeIDs = zeros(Int, num_interfaces) + neighIDs = zeros(Int, num_interfaces) + orientations = zeros(Int32, num_interfaces) + faces = zeros(Int8, num_interfaces) + duals = zeros(Int8, num_interfaces) - sfc_index = 0 # space-filling curve index itf_index = 0 # interface index - # Loop over all local trees. - for itree in 0:(num_local_trees - 1) - tree_class = t8_forest_get_tree_class(forest, itree) - eclass_scheme = t8_forest_get_eclass_scheme(forest, tree_class) - - num_elements_in_tree = t8_forest_get_tree_num_elements(forest, itree) - - # Loop over all local elements of the current local tree. - for ielement in 0:(num_elements_in_tree - 1) - element = t8_forest_get_element_in_tree(forest, itree, ielement) - level = t8_element_level(eclass_scheme, element) - num_faces = t8_element_num_faces(eclass_scheme, element) - - # Loop over all faces of the current local element. - for iface in 0:(num_faces - 1) - # Compute the `orientation` of the touching faces. - if t8_element_is_root_boundary(eclass_scheme, element, iface) == 1 - itree_in_cmesh = t8_forest_ltreeid_to_cmesh_ltreeid(forest, itree) - iface_in_tree = t8_element_tree_face(eclass_scheme, element, iface) - orientation_ref = Ref{Cint}() - - t8_cmesh_get_face_neighbor(cmesh, itree_in_cmesh, iface_in_tree, C_NULL, - orientation_ref) - orientation = orientation_ref[] - else - orientation = zero(Cint) - end - - pelement_indices_ref = Ref{Ptr{t8_locidx_t}}() - pneighbor_leaves_ref = Ref{Ptr{Ptr{t8_element}}}() - pneigh_scheme_ref = Ref{Ptr{t8_eclass_scheme}}() - - dual_faces_ref = Ref{Ptr{Cint}}() - num_neighbors_ref = Ref{Cint}() - - forest_is_balanced = Cint(1) - - # Query neighbor information from t8code. - t8_forest_leaf_face_neighbors(forest, itree, element, - pneighbor_leaves_ref, iface, dual_faces_ref, - num_neighbors_ref, - pelement_indices_ref, pneigh_scheme_ref, - forest_is_balanced) - - num_neighbors = num_neighbors_ref[] - dual_faces = unsafe_wrap(Array, dual_faces_ref[], num_neighbors) - neighbor_ielements = unsafe_wrap(Array, pelement_indices_ref[], num_neighbors) - neighbor_leaves = unsafe_wrap(Array, pneighbor_leaves_ref[], num_neighbors) - neighbor_scheme = pneigh_scheme_ref[] - - # The condition ensures we only visit the interface once. - if num_neighbors == 1 && sfc_index <= neighbor_ielements[1] - itf_index += 1 - elemIDs[itf_index] = sfc_index - neighIDs[itf_index] = neighbor_ielements[1] - orientations[itf_index] = orientation - faces[itf_index] = iface - duals[itf_index] = dual_faces[1] - end - - t8_free(dual_faces_ref[]) - t8_free(pneighbor_leaves_ref[]) - t8_free(pelement_indices_ref[]) - end # for iface - - sfc_index += 1 - end # for ielement - end # for itree - - t8_forest_unref(Ref(forest)) + for itree in 0:(num_trees - 1) + for iface in 0:(num_faces - 1) + neigh_itree = t8_cmesh_get_face_neighbor(cmesh, itree, iface, dual_face_ref, orientation_ref) + + if itree < neigh_itree || itree == neigh_itree && iface < dual_face_ref[] + itf_index += 1 + treeIDs[itf_index] = itree + neighIDs[itf_index] = neigh_itree + orientations[itf_index] = orientation_ref[] + faces[itf_index] = iface + duals[itf_index] = dual_face_ref[] + end + end + end - return elemIDs, neighIDs, faces, duals, orientations + return treeIDs, neighIDs, faces, duals, orientations end #! format: off From 2f5d22441b451c0fe9af5011751d45c0c7130f33 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Wed, 19 Jun 2024 17:47:15 +0200 Subject: [PATCH 48/81] Added support for parallelt8codemesh save solution callback. --- src/callbacks_step/save_solution_dg.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/callbacks_step/save_solution_dg.jl b/src/callbacks_step/save_solution_dg.jl index deae8f7c930..555f41dbd66 100644 --- a/src/callbacks_step/save_solution_dg.jl +++ b/src/callbacks_step/save_solution_dg.jl @@ -91,7 +91,7 @@ function save_solution_file(u, time, dt, timestep, end function save_solution_file(u, time, dt, timestep, - mesh::Union{ParallelTreeMesh, ParallelP4estMesh}, equations, + mesh::Union{ParallelTreeMesh, ParallelP4estMesh, ParallelT8codeMesh}, equations, dg::DG, cache, solution_callback, element_variables = Dict{Symbol, Any}(), @@ -136,7 +136,7 @@ function save_solution_file(u, time, dt, timestep, end function save_solution_file_parallel(data, time, dt, timestep, n_vars, - mesh::Union{ParallelTreeMesh, ParallelP4estMesh}, + mesh::Union{ParallelTreeMesh, ParallelP4estMesh, ParallelT8codeMesh}, equations, dg::DG, cache, solution_variables, filename, element_variables = Dict{Symbol, Any}()) From d3a1b4958b5e1326379c5eb9c71c01ce5786c4e4 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Wed, 19 Jun 2024 17:49:10 +0200 Subject: [PATCH 49/81] Applied formatter. --- .../elixir_advection_unstructured_flag.jl | 3 +- src/auxiliary/t8code.jl | 2 +- src/callbacks_step/save_solution_dg.jl | 6 +- src/meshes/mesh_io.jl | 9 +- src/meshes/t8code_mesh.jl | 117 ++++++++++-------- 5 files changed, 73 insertions(+), 64 deletions(-) diff --git a/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl b/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl index d966ef2273d..aed6c755d5c 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl @@ -63,7 +63,8 @@ stepsize_callback = StepsizeCallback(cfl = 1.4) # Create a CallbackSet to collect all callbacks such that they can be passed to # the ODE solver. -callbacks = CallbackSet(summary_callback, analysis_callback, save_solution, stepsize_callback) +callbacks = CallbackSet(summary_callback, analysis_callback, save_solution, + stepsize_callback) ############################################################################### # Run the simulation. diff --git a/src/auxiliary/t8code.jl b/src/auxiliary/t8code.jl index 9017d8fb851..83ef471e1c6 100644 --- a/src/auxiliary/t8code.jl +++ b/src/auxiliary/t8code.jl @@ -18,7 +18,7 @@ function init_t8code() let catch_signals = 0, print_backtrace = 0, log_handler = C_NULL T8code.Libt8.sc_init(mpi_comm(), catch_signals, print_backtrace, log_handler, T8code.Libt8.SC_LP_DEBUG) - # T8code.Libt8.SC_LP_ERROR) + # T8code.Libt8.SC_LP_ERROR) end if T8code.Libt8.p4est_is_initialized() == 0 diff --git a/src/callbacks_step/save_solution_dg.jl b/src/callbacks_step/save_solution_dg.jl index 555f41dbd66..33e4997ca93 100644 --- a/src/callbacks_step/save_solution_dg.jl +++ b/src/callbacks_step/save_solution_dg.jl @@ -91,7 +91,8 @@ function save_solution_file(u, time, dt, timestep, end function save_solution_file(u, time, dt, timestep, - mesh::Union{ParallelTreeMesh, ParallelP4estMesh, ParallelT8codeMesh}, equations, + mesh::Union{ParallelTreeMesh, ParallelP4estMesh, + ParallelT8codeMesh}, equations, dg::DG, cache, solution_callback, element_variables = Dict{Symbol, Any}(), @@ -136,7 +137,8 @@ function save_solution_file(u, time, dt, timestep, end function save_solution_file_parallel(data, time, dt, timestep, n_vars, - mesh::Union{ParallelTreeMesh, ParallelP4estMesh, ParallelT8codeMesh}, + mesh::Union{ParallelTreeMesh, ParallelP4estMesh, + ParallelT8codeMesh}, equations, dg::DG, cache, solution_variables, filename, element_variables = Dict{Symbol, Any}()) diff --git a/src/meshes/mesh_io.jl b/src/meshes/mesh_io.jl index 31c73ef6537..8958936f771 100644 --- a/src/meshes/mesh_io.jl +++ b/src/meshes/mesh_io.jl @@ -241,7 +241,7 @@ function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, # Retrieve refinement levels of all elements. levels = get_levels(mesh) if mpi_isparallel() - levels = MPI.Gather(levels, mpi_root(), mpi_comm()) + levels = MPI.Gather(levels, mpi_root(), mpi_comm()) end # Retrieve number of elements per tree. @@ -249,13 +249,14 @@ function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, num_elements_per_tree = zeros(t8_gloidx_t, num_global_trees) num_local_trees = t8_forest_get_num_local_trees(mesh.forest) for local_tree_id in 0:(num_local_trees - 1) - num_local_elements_in_tree = t8_forest_get_tree_num_elements(mesh.forest, local_tree_id) + num_local_elements_in_tree = t8_forest_get_tree_num_elements(mesh.forest, + local_tree_id) global_tree_id = t8_forest_global_tree_id(mesh.forest, local_tree_id) num_elements_per_tree[global_tree_id + 1] = num_local_elements_in_tree end if mpi_isparallel() - num_elements_per_tree = MPI.Reduce!(num_elements_per_tree, +, mpi_comm()) + num_elements_per_tree = MPI.Reduce!(num_elements_per_tree, +, mpi_comm()) end # Since the mesh attributes are replicated on all ranks, only save from MPI @@ -406,7 +407,6 @@ function load_mesh_serial(mesh_file::AbstractString; n_cells_max, RealT) nodes, boundary_names, treeIDs, neighIDs, faces, duals, orientations, levels, num_elements_per_tree) else - error("Unknown mesh type!") end @@ -597,5 +597,4 @@ function load_mesh!(mesh::ParallelTreeMesh, mesh_file::AbstractString) return mesh end - end # @muladd diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 9fe6836db54..d75741ca995 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -26,7 +26,7 @@ mutable struct T8codeMesh{NDIMS, RealT <: Real, IsParallel, NDIMSP2, NNODES} <: nmpiinterfaces :: Int nmpimortars :: Int - unsaved_changes :: Bool + unsaved_changes::Bool function T8codeMesh{NDIMS}(forest::Ptr{t8_forest}, tree_node_coordinates, nodes, boundary_names, @@ -139,62 +139,63 @@ function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, # Store element vertices inside the cmesh. N = length(nodes) vertices = zeros(3 * 2^ndims) # quads/hexs only - for i = 1:ntrees - t8_cmesh_set_tree_class(cmesh, i-1, eclass) + for i in 1:ntrees + t8_cmesh_set_tree_class(cmesh, i - 1, eclass) if ndims == 2 - vertices[1] = tree_node_coordinates[1,1,1,i] - vertices[2] = tree_node_coordinates[2,1,1,i] + vertices[1] = tree_node_coordinates[1, 1, 1, i] + vertices[2] = tree_node_coordinates[2, 1, 1, i] - vertices[4] = tree_node_coordinates[1,N,1,i] - vertices[5] = tree_node_coordinates[2,N,1,i] + vertices[4] = tree_node_coordinates[1, N, 1, i] + vertices[5] = tree_node_coordinates[2, N, 1, i] - vertices[7] = tree_node_coordinates[1,1,N,i] - vertices[8] = tree_node_coordinates[2,1,N,i] + vertices[7] = tree_node_coordinates[1, 1, N, i] + vertices[8] = tree_node_coordinates[2, 1, N, i] - vertices[10] = tree_node_coordinates[1,N,N,i] - vertices[11] = tree_node_coordinates[2,N,N,i] + vertices[10] = tree_node_coordinates[1, N, N, i] + vertices[11] = tree_node_coordinates[2, N, N, i] else - vertices[1] = tree_node_coordinates[1,1,1,1,i] - vertices[2] = tree_node_coordinates[2,1,1,1,i] - vertices[3] = tree_node_coordinates[3,1,1,1,i] + vertices[1] = tree_node_coordinates[1, 1, 1, 1, i] + vertices[2] = tree_node_coordinates[2, 1, 1, 1, i] + vertices[3] = tree_node_coordinates[3, 1, 1, 1, i] - vertices[4] = tree_node_coordinates[1,N,1,1,i] - vertices[5] = tree_node_coordinates[2,N,1,1,i] - vertices[6] = tree_node_coordinates[3,N,1,1,i] + vertices[4] = tree_node_coordinates[1, N, 1, 1, i] + vertices[5] = tree_node_coordinates[2, N, 1, 1, i] + vertices[6] = tree_node_coordinates[3, N, 1, 1, i] - vertices[7] = tree_node_coordinates[1,1,N,1,i] - vertices[8] = tree_node_coordinates[2,1,N,1,i] - vertices[9] = tree_node_coordinates[3,1,N,1,i] + vertices[7] = tree_node_coordinates[1, 1, N, 1, i] + vertices[8] = tree_node_coordinates[2, 1, N, 1, i] + vertices[9] = tree_node_coordinates[3, 1, N, 1, i] - vertices[10] = tree_node_coordinates[1,N,N,1,i] - vertices[11] = tree_node_coordinates[2,N,N,1,i] - vertices[12] = tree_node_coordinates[3,N,N,1,i] + vertices[10] = tree_node_coordinates[1, N, N, 1, i] + vertices[11] = tree_node_coordinates[2, N, N, 1, i] + vertices[12] = tree_node_coordinates[3, N, N, 1, i] - vertices[13] = tree_node_coordinates[1,1,1,N,i] - vertices[14] = tree_node_coordinates[2,1,1,N,i] - vertices[15] = tree_node_coordinates[3,1,1,N,i] + vertices[13] = tree_node_coordinates[1, 1, 1, N, i] + vertices[14] = tree_node_coordinates[2, 1, 1, N, i] + vertices[15] = tree_node_coordinates[3, 1, 1, N, i] - vertices[16] = tree_node_coordinates[1,N,1,N,i] - vertices[17] = tree_node_coordinates[2,N,1,N,i] - vertices[18] = tree_node_coordinates[3,N,1,N,i] + vertices[16] = tree_node_coordinates[1, N, 1, N, i] + vertices[17] = tree_node_coordinates[2, N, 1, N, i] + vertices[18] = tree_node_coordinates[3, N, 1, N, i] - vertices[19] = tree_node_coordinates[1,1,N,N,i] - vertices[20] = tree_node_coordinates[2,1,N,N,i] - vertices[21] = tree_node_coordinates[3,1,N,N,i] + vertices[19] = tree_node_coordinates[1, 1, N, N, i] + vertices[20] = tree_node_coordinates[2, 1, N, N, i] + vertices[21] = tree_node_coordinates[3, 1, N, N, i] - vertices[22] = tree_node_coordinates[1,N,N,N,i] - vertices[23] = tree_node_coordinates[2,N,N,N,i] - vertices[24] = tree_node_coordinates[3,N,N,N,i] + vertices[22] = tree_node_coordinates[1, N, N, N, i] + vertices[23] = tree_node_coordinates[2, N, N, N, i] + vertices[24] = tree_node_coordinates[3, N, N, N, i] end - t8_cmesh_set_tree_vertices(cmesh, i-1, vertices, 3) + t8_cmesh_set_tree_vertices(cmesh, i - 1, vertices, 3) end # Connect the coarse mesh elements. - for i = 1:length(elemIDs) - t8_cmesh_set_join(cmesh, elemIDs[i], neighIDs[i], faces[i], duals[i], orientations[i]) - end + for i in 1:length(elemIDs) + t8_cmesh_set_join(cmesh, elemIDs[i], neighIDs[i], faces[i], duals[i], + orientations[i]) + end t8_cmesh_commit(cmesh, mpi_comm()) @@ -202,7 +203,8 @@ function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, do_face_ghost = mpi_isparallel() scheme = t8_scheme_new_default_cxx() initial_refinement_level = 0 - forest = t8_forest_new_uniform(cmesh, scheme, initial_refinement_level, do_face_ghost, mpi_comm()) + forest = t8_forest_new_uniform(cmesh, scheme, initial_refinement_level, do_face_ghost, + mpi_comm()) cum_sum_num_elements_per_tree = cumsum(num_elements_per_tree) @@ -214,19 +216,20 @@ function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, virtual_element_index += cum_sum_num_elements_per_tree[last_global_tree_id_of_preceding_rank + 1] end - function adapt_callback(forest, local_tree_id, eclass_scheme, local_element_id, elements, is_family, + function adapt_callback(forest, local_tree_id, eclass_scheme, local_element_id, + elements, is_family, user_data) # Check if we are already in the next tree in terms of the `virtual_element_index`. global_tree_id = t8_forest_global_tree_id(forest, local_tree_id) if virtual_element_index > cum_sum_num_elements_per_tree[global_tree_id + 1] - return 0 + return 0 end # Test if we already reached the targeted level. level = t8_element_level(eclass_scheme, elements[1]) if level < levels[virtual_element_index] - return 1 # Go one refinement level deeper. + return 1 # Go one refinement level deeper. end # Targeted level is reached. @@ -238,7 +241,7 @@ function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, # For each tree the callback recursively increases the refinement level # till it matches with the associated section in `levels. forest = adapt(forest, adapt_callback; recursive = true, balance = false, - partition = false, ghost = false, user_data = C_NULL) + partition = false, ghost = false, user_data = C_NULL) # if mpi_isparallel() # forest = t8_forest_partition(forest) @@ -849,7 +852,7 @@ Adapt a `T8codeMesh` according to a user-defined `adapt_callback`. - `user_data = C_NULL`: Pointer to some arbitrary user-defined data. """ function adapt(forest::Ptr{t8_forest}, adapt_callback; recursive = true, balance = true, - partition = true, ghost = true, user_data = C_NULL) + partition = true, ghost = true, user_data = C_NULL) # Check that forest is a committed, that is valid and usable, forest. @assert t8_forest_is_committed(forest) != 0 @@ -945,7 +948,7 @@ function get_global_first_element_ids(mesh::T8codeMesh) end function count_interfaces(mesh::T8codeMesh) - return count_interfaces(mesh.forest,ndims(mesh)) + return count_interfaces(mesh.forest, ndims(mesh)) end function count_interfaces(forest::Ptr{t8_forest}, ndims) @@ -1235,11 +1238,13 @@ function fill_mesh_info!(mesh::T8codeMesh, interfaces, mortars, boundaries, # Compute the `orientation` of the touching faces. if t8_element_is_root_boundary(eclass_scheme, element, iface) == 1 - itree_in_cmesh = t8_forest_ltreeid_to_cmesh_ltreeid(mesh.forest, itree) + itree_in_cmesh = t8_forest_ltreeid_to_cmesh_ltreeid(mesh.forest, + itree) iface_in_tree = t8_element_tree_face(eclass_scheme, element, iface) orientation_ref = Ref{Cint}() - t8_cmesh_get_face_neighbor(cmesh, itree_in_cmesh, iface_in_tree, C_NULL, + t8_cmesh_get_face_neighbor(cmesh, itree_in_cmesh, iface_in_tree, + C_NULL, orientation_ref) orientation = orientation_ref[] else @@ -1418,18 +1423,18 @@ function fill_mesh_info!(mesh::T8codeMesh, interfaces, mortars, boundaries, end function get_levels(mesh::T8codeMesh) - return trixi_t8_get_local_element_levels(mesh.forest) + return trixi_t8_get_local_element_levels(mesh.forest) end function get_cmesh_info(mesh::T8codeMesh) - @assert t8_forest_is_committed(mesh.forest) != 0 - cmesh = t8_forest_get_cmesh(mesh.forest) - return get_cmesh_info(cmesh, ndims(mesh)) + @assert t8_forest_is_committed(mesh.forest) != 0 + cmesh = t8_forest_get_cmesh(mesh.forest) + return get_cmesh_info(cmesh, ndims(mesh)) end function get_cmesh_info(cmesh::Ptr{t8_cmesh}, ndims) num_trees = t8_cmesh_get_num_trees(cmesh) - num_faces = 2*ndims + num_faces = 2 * ndims num_interfaces = 0 @@ -1439,7 +1444,8 @@ function get_cmesh_info(cmesh::Ptr{t8_cmesh}, ndims) # Count connected faces. for itree in 0:(num_trees - 1) for iface in 0:(num_faces - 1) - neigh_itree = t8_cmesh_get_face_neighbor(cmesh, itree, iface, dual_face_ref, C_NULL) + neigh_itree = t8_cmesh_get_face_neighbor(cmesh, itree, iface, dual_face_ref, + C_NULL) if itree < neigh_itree || itree == neigh_itree && iface < dual_face_ref[] num_interfaces += 1 end @@ -1457,7 +1463,8 @@ function get_cmesh_info(cmesh::Ptr{t8_cmesh}, ndims) for itree in 0:(num_trees - 1) for iface in 0:(num_faces - 1) - neigh_itree = t8_cmesh_get_face_neighbor(cmesh, itree, iface, dual_face_ref, orientation_ref) + neigh_itree = t8_cmesh_get_face_neighbor(cmesh, itree, iface, dual_face_ref, + orientation_ref) if itree < neigh_itree || itree == neigh_itree && iface < dual_face_ref[] itf_index += 1 From 9ef91c6d04c1e628d398a5dade1142c3e7bb5609 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Thu, 20 Jun 2024 14:53:12 +0200 Subject: [PATCH 50/81] Updated examples and tests. --- ...ixir_advection_amr_solution_independent.jl | 6 ++ .../elixir_advection_amr_unstructured_flag.jl | 9 ++ .../elixir_advection_extended.jl | 85 +++++++++++++++++++ .../elixir_advection_nonconforming_flag.jl | 6 +- .../elixir_advection_restart.jl | 43 ++++++++++ .../elixir_advection_restart_amr.jl | 54 ++++++++++++ .../elixir_euler_free_stream.jl | 6 ++ .../t8code_2d_dgsem/elixir_euler_sedov.jl | 5 ++ .../elixir_euler_shockcapturing_ec.jl | 6 ++ ...e_terms_nonconforming_unstructured_flag.jl | 9 ++ .../elixir_eulergravity_convergence.jl | 9 ++ examples/t8code_2d_dgsem/elixir_mhd_rotor.jl | 6 ++ .../elixir_shallowwater_source_terms.jl | 6 +- .../t8code_3d_dgsem/elixir_advection_amr.jl | 6 ++ ...lixir_advection_amr_unstructured_curved.jl | 10 +++ .../t8code_3d_dgsem/elixir_advection_basic.jl | 10 ++- .../elixir_advection_nonconforming.jl | 6 +- .../elixir_advection_restart.jl | 41 +++++++++ .../elixir_advection_unstructured_curved.jl | 10 ++- examples/t8code_3d_dgsem/elixir_euler_ec.jl | 5 ++ .../elixir_euler_free_stream.jl | 6 ++ .../elixir_euler_free_stream_extruded.jl | 6 ++ .../t8code_3d_dgsem/elixir_euler_sedov.jl | 5 ++ ...terms_nonconforming_unstructured_curved.jl | 6 ++ .../elixir_euler_source_terms_nonperiodic.jl | 6 ++ src/auxiliary/t8code.jl | 6 +- src/callbacks_step/save_restart_dg.jl | 14 +-- test/test_mpi_t8code_2d.jl | 18 ++++ test/test_mpi_t8code_3d.jl | 18 ++++ test/test_t8code_2d.jl | 46 +++++++--- test/test_t8code_3d.jl | 18 ++++ 31 files changed, 455 insertions(+), 32 deletions(-) create mode 100644 examples/t8code_2d_dgsem/elixir_advection_extended.jl create mode 100644 examples/t8code_2d_dgsem/elixir_advection_restart.jl create mode 100644 examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl create mode 100644 examples/t8code_3d_dgsem/elixir_advection_restart.jl diff --git a/examples/t8code_2d_dgsem/elixir_advection_amr_solution_independent.jl b/examples/t8code_2d_dgsem/elixir_advection_amr_solution_independent.jl index 1ed08e1961b..d10cb167cb0 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_amr_solution_independent.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_amr_solution_independent.jl @@ -115,6 +115,11 @@ analysis_callback = AnalysisCallback(semi, interval = analysis_interval, alive_callback = AliveCallback(analysis_interval = analysis_interval) +save_solution = SaveSolutionCallback(interval = 100, + save_initial_solution = true, + save_final_solution = true, + solution_variables = cons2prim) + amr_controller = ControllerThreeLevel(semi, TrixiExtension.IndicatorSolutionIndependent(semi), base_level = 4, @@ -130,6 +135,7 @@ stepsize_callback = StepsizeCallback(cfl = 1.6) callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, + save_solution, amr_callback, stepsize_callback); ############################################################################### diff --git a/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl b/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl index 9138586cccf..352455d986f 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl @@ -54,6 +54,14 @@ analysis_callback = AnalysisCallback(semi, interval = analysis_interval, alive_callback = AliveCallback(analysis_interval = analysis_interval) +save_restart = SaveRestartCallback(interval = 100, + save_final_restart = true) + +save_solution = SaveSolutionCallback(interval = 100, + save_initial_solution = true, + save_final_solution = true, + solution_variables = cons2prim) + amr_controller = ControllerThreeLevel(semi, IndicatorMax(semi, variable = first), base_level = 1, med_level = 2, med_threshold = 0.1, @@ -67,6 +75,7 @@ stepsize_callback = StepsizeCallback(cfl = 0.7) callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, + save_restart, save_solution, amr_callback, stepsize_callback) ############################################################################### diff --git a/examples/t8code_2d_dgsem/elixir_advection_extended.jl b/examples/t8code_2d_dgsem/elixir_advection_extended.jl new file mode 100644 index 00000000000..44892faf40d --- /dev/null +++ b/examples/t8code_2d_dgsem/elixir_advection_extended.jl @@ -0,0 +1,85 @@ + +using OrdinaryDiffEq +using Trixi + +############################################################################### +# semidiscretization of the linear advection equation + +advection_velocity = (0.2, -0.7) +equations = LinearScalarAdvectionEquation2D(advection_velocity) + +initial_condition = initial_condition_convergence_test + +# BCs must be passed as Dict +boundary_condition = BoundaryConditionDirichlet(initial_condition) +boundary_conditions = Dict(:x_neg => boundary_condition, + :x_pos => boundary_condition, + :y_neg => boundary_condition, + :y_pos => boundary_condition) + +# Create DG solver with polynomial degree = 3 and (local) Lax-Friedrichs/Rusanov flux as surface flux +solver = DGSEM(polydeg = 3, surface_flux = flux_lax_friedrichs) + +# The initial condition is 2-periodic +coordinates_min = (-1.5, 1.3) # minimum coordinates (min(x), min(y)) +coordinates_max = (0.5, 5.3) # maximum coordinates (max(x), max(y)) + +trees_per_dimension = (19, 37) + +# Create curved mesh with 19 x 37 elements +mesh = T8codeMesh(trees_per_dimension, polydeg = 3, + coordinates_min = coordinates_min, coordinates_max = coordinates_max, + periodicity = false) + +# A semidiscretization collects data structures and functions for the spatial discretization +semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition, solver, + boundary_conditions = boundary_conditions) + +############################################################################### +# ODE solvers, callbacks etc. + +# Create ODE problem with time span from 0.0 to 1.0 +tspan = (0.0, 1.0) +ode = semidiscretize(semi, tspan); + +# At the beginning of the main loop, the SummaryCallback prints a summary of the simulation setup +# and resets the timers +summary_callback = SummaryCallback() + +# The AnalysisCallback allows to analyse the solution in regular intervals and prints the results +analysis_interval = 100 +analysis_callback = AnalysisCallback(semi, interval = analysis_interval, + extra_analysis_integrals = (entropy, energy_total)) + +# The AliveCallback prints short status information in regular intervals +alive_callback = AliveCallback(analysis_interval = analysis_interval) + +# The SaveRestartCallback allows to save a file from which a Trixi.jl simulation can be restarted +save_restart = SaveRestartCallback(interval = 100, + save_final_restart = true) + +# The SaveSolutionCallback allows to save the solution to a file in regular intervals +save_solution = SaveSolutionCallback(interval = 100, + save_initial_solution = true, + save_final_solution = true, + solution_variables = cons2prim) + +# The StepsizeCallback handles the re-calculation of the maximum Δt after each time step +stepsize_callback = StepsizeCallback(cfl = 1.6) + +# Create a CallbackSet to collect all callbacks such that they can be passed to the ODE solver +callbacks = CallbackSet(summary_callback, + analysis_callback, alive_callback, + save_restart, save_solution, + stepsize_callback) + +############################################################################### +# run the simulation + +# OrdinaryDiffEq's `solve` method evolves the solution in time and executes the passed callbacks +sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), + dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback + save_everystep = false, callback = callbacks); + +# Print the timer summary +summary_callback() diff --git a/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl b/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl index 48f78dd6da3..f080b640f0d 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl @@ -69,11 +69,15 @@ summary_callback = SummaryCallback() # The AnalysisCallback allows to analyse the solution in regular intervals and prints the results analysis_callback = AnalysisCallback(semi, interval = 100) +# The SaveSolutionCallback allows to save the solution to a file in regular intervals +save_solution = SaveSolutionCallback(interval = 100, + solution_variables = cons2prim) + # The StepsizeCallback handles the re-calculation of the maximum Δt after each time step stepsize_callback = StepsizeCallback(cfl = 1.6) # Create a CallbackSet to collect all callbacks such that they can be passed to the ODE solver -callbacks = CallbackSet(summary_callback, analysis_callback, stepsize_callback) +callbacks = CallbackSet(summary_callback, analysis_callback, save_solution, stepsize_callback) ############################################################################### # run the simulation diff --git a/examples/t8code_2d_dgsem/elixir_advection_restart.jl b/examples/t8code_2d_dgsem/elixir_advection_restart.jl new file mode 100644 index 00000000000..0f573714c1f --- /dev/null +++ b/examples/t8code_2d_dgsem/elixir_advection_restart.jl @@ -0,0 +1,43 @@ + +using OrdinaryDiffEq +using Trixi + +############################################################################### +# create a restart file + +elixir_file = "elixir_advection_extended.jl" +restart_file = "restart_000021.h5" + +trixi_include(@__MODULE__, joinpath(@__DIR__, elixir_file)) + +############################################################################### +# adapt the parameters that have changed compared to "elixir_advection_extended.jl" + +# Note: If you get a restart file from somewhere else, you need to provide +# appropriate setups in the elixir loading a restart file + +restart_filename = joinpath("out", restart_file) +mesh = load_mesh(restart_filename) + +semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition, solver, + boundary_conditions = boundary_conditions) + +tspan = (load_time(restart_filename), 2.0) +dt = load_dt(restart_filename) +ode = semidiscretize(semi, tspan, restart_filename); + +# Do not overwrite the initial snapshot written by elixir_advection_extended.jl. +save_solution.condition.save_initial_solution = false + +integrator = init(ode, CarpenterKennedy2N54(williamson_condition = false), + dt = dt, # solve needs some value here but it will be overwritten by the stepsize_callback + save_everystep = false, callback = callbacks, maxiters = 100_000); + +# Get the last time index and work with that. +load_timestep!(integrator, restart_filename) + +############################################################################### +# run the simulation + +sol = solve!(integrator) +summary_callback() # print the timer summary diff --git a/examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl b/examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl new file mode 100644 index 00000000000..fd3623dd88b --- /dev/null +++ b/examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl @@ -0,0 +1,54 @@ + +using OrdinaryDiffEq +using Trixi + +############################################################################### +# create a restart file + +elixir_file = "elixir_advection_extended.jl" +restart_file = "restart_000021.h5" + +trixi_include(@__MODULE__, joinpath(@__DIR__, elixir_file)) + +############################################################################### +# adapt the parameters that have changed compared to "elixir_advection_extended.jl" + +# Note: If you get a restart file from somewhere else, you need to provide +# appropriate setups in the elixir loading a restart file + +restart_filename = joinpath("out", restart_file) +mesh = load_mesh(restart_filename) + +semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition, solver, + boundary_conditions = boundary_conditions) + +tspan = (load_time(restart_filename), 2.0) +dt = load_dt(restart_filename) +ode = semidiscretize(semi, tspan, restart_filename); + +# Do not overwrite the initial snapshot written by elixir_advection_extended.jl. +save_solution.condition.save_initial_solution = false + +# Add AMR callback +amr_controller = ControllerThreeLevel(semi, IndicatorMax(semi, variable = first), + base_level = 0, + med_level = 0, med_threshold = 0.8, + max_level = 1, max_threshold = 1.2) +amr_callback = AMRCallback(semi, amr_controller, + interval = 5, + adapt_initial_condition = true, + adapt_initial_condition_only_refine = true) +callbacks_ext = CallbackSet(amr_callback, callbacks.discrete_callbacks...) + +integrator = init(ode, CarpenterKennedy2N54(williamson_condition = false), + dt = dt, # solve needs some value here but it will be overwritten by the stepsize_callback + save_everystep = false, callback = callbacks_ext, maxiters = 100_000); + +# Get the last time index and work with that. +load_timestep!(integrator, restart_filename) + +############################################################################### +# run the simulation + +sol = solve!(integrator) +summary_callback() # print the timer summary diff --git a/examples/t8code_2d_dgsem/elixir_euler_free_stream.jl b/examples/t8code_2d_dgsem/elixir_euler_free_stream.jl index d9d2c65d988..b2d49e3ccfe 100644 --- a/examples/t8code_2d_dgsem/elixir_euler_free_stream.jl +++ b/examples/t8code_2d_dgsem/elixir_euler_free_stream.jl @@ -72,10 +72,16 @@ analysis_callback = AnalysisCallback(semi, interval = analysis_interval) alive_callback = AliveCallback(analysis_interval = analysis_interval) +save_solution = SaveSolutionCallback(interval = 100, + save_initial_solution = true, + save_final_solution = true, + solution_variables = cons2prim) + stepsize_callback = StepsizeCallback(cfl = 2.0) callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, + save_solution, stepsize_callback) ############################################################################### diff --git a/examples/t8code_2d_dgsem/elixir_euler_sedov.jl b/examples/t8code_2d_dgsem/elixir_euler_sedov.jl index 82770a4050b..fae7d818ad8 100644 --- a/examples/t8code_2d_dgsem/elixir_euler_sedov.jl +++ b/examples/t8code_2d_dgsem/elixir_euler_sedov.jl @@ -79,11 +79,16 @@ analysis_callback = AnalysisCallback(semi, interval = analysis_interval) alive_callback = AliveCallback(analysis_interval = analysis_interval) +save_solution = SaveSolutionCallback(interval = 300, + save_initial_solution = true, + save_final_solution = true) + stepsize_callback = StepsizeCallback(cfl = 0.5) callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, + save_solution, stepsize_callback) ############################################################################### diff --git a/examples/t8code_2d_dgsem/elixir_euler_shockcapturing_ec.jl b/examples/t8code_2d_dgsem/elixir_euler_shockcapturing_ec.jl index 9ebbd1d28c4..2a0c4a6ee20 100644 --- a/examples/t8code_2d_dgsem/elixir_euler_shockcapturing_ec.jl +++ b/examples/t8code_2d_dgsem/elixir_euler_shockcapturing_ec.jl @@ -50,11 +50,17 @@ analysis_callback = AnalysisCallback(semi, interval = analysis_interval) alive_callback = AliveCallback(analysis_interval = analysis_interval) +save_solution = SaveSolutionCallback(interval = 100, + save_initial_solution = true, + save_final_solution = true, + solution_variables = cons2prim) + stepsize_callback = StepsizeCallback(cfl = 1.0) callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, + save_solution, stepsize_callback) ############################################################################### diff --git a/examples/t8code_2d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_flag.jl b/examples/t8code_2d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_flag.jl index 48684071d4b..19502f1ce0e 100644 --- a/examples/t8code_2d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_flag.jl @@ -73,10 +73,19 @@ analysis_callback = AnalysisCallback(semi, interval = analysis_interval) alive_callback = AliveCallback(analysis_interval = analysis_interval) +save_restart = SaveRestartCallback(interval = 100, + save_final_restart = true) + +save_solution = SaveSolutionCallback(interval = 100, + save_initial_solution = true, + save_final_solution = true, + solution_variables = cons2prim) + stepsize_callback = StepsizeCallback(cfl = 0.8) callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, + save_restart, save_solution, stepsize_callback) ############################################################################### # run the simulation diff --git a/examples/t8code_2d_dgsem/elixir_eulergravity_convergence.jl b/examples/t8code_2d_dgsem/elixir_eulergravity_convergence.jl index cd10315945a..e63c1297882 100644 --- a/examples/t8code_2d_dgsem/elixir_eulergravity_convergence.jl +++ b/examples/t8code_2d_dgsem/elixir_eulergravity_convergence.jl @@ -56,6 +56,14 @@ summary_callback = SummaryCallback() stepsize_callback = StepsizeCallback(cfl = 0.8) +save_solution = SaveSolutionCallback(interval = 10, + save_initial_solution = true, + save_final_solution = true, + solution_variables = cons2prim) + +save_restart = SaveRestartCallback(interval = 100, + save_final_restart = true) + analysis_interval = 100 alive_callback = AliveCallback(analysis_interval = analysis_interval) @@ -63,6 +71,7 @@ analysis_callback = AnalysisCallback(semi_euler, interval = analysis_interval, save_analysis = true) callbacks = CallbackSet(summary_callback, stepsize_callback, + save_restart, save_solution, analysis_callback, alive_callback) ############################################################################### diff --git a/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl b/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl index 592d5b15a85..a09ea46fa34 100644 --- a/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl +++ b/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl @@ -93,6 +93,11 @@ analysis_callback = AnalysisCallback(semi, interval = analysis_interval) alive_callback = AliveCallback(analysis_interval = analysis_interval) +save_solution = SaveSolutionCallback(interval = 100, + save_initial_solution = true, + save_final_solution = true, + solution_variables = cons2prim) + amr_indicator = IndicatorLöhner(semi, variable = density_pressure) @@ -113,6 +118,7 @@ glm_speed_callback = GlmSpeedCallback(glm_scale = 0.5, cfl = cfl) callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, + save_solution, amr_callback, stepsize_callback, glm_speed_callback) diff --git a/examples/t8code_2d_dgsem/elixir_shallowwater_source_terms.jl b/examples/t8code_2d_dgsem/elixir_shallowwater_source_terms.jl index 3610639d554..688ddb2dbb5 100644 --- a/examples/t8code_2d_dgsem/elixir_shallowwater_source_terms.jl +++ b/examples/t8code_2d_dgsem/elixir_shallowwater_source_terms.jl @@ -46,7 +46,11 @@ analysis_callback = AnalysisCallback(semi, interval = analysis_interval) alive_callback = AliveCallback(analysis_interval = analysis_interval) -callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback) +save_solution = SaveSolutionCallback(interval = 200, + save_initial_solution = true, + save_final_solution = true) + +callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, save_solution) ############################################################################### # run the simulation diff --git a/examples/t8code_3d_dgsem/elixir_advection_amr.jl b/examples/t8code_3d_dgsem/elixir_advection_amr.jl index 5a4b2218d57..9ecc8955383 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_amr.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_amr.jl @@ -40,6 +40,11 @@ analysis_callback = AnalysisCallback(semi, interval = analysis_interval, alive_callback = AliveCallback(analysis_interval = analysis_interval) +save_solution = SaveSolutionCallback(interval = 100, + save_initial_solution = true, + save_final_solution = true, + solution_variables = cons2prim) + amr_controller = ControllerThreeLevel(semi, IndicatorMax(semi, variable = first), base_level = 4, med_level = 5, med_threshold = 0.1, @@ -54,6 +59,7 @@ stepsize_callback = StepsizeCallback(cfl = 1.2) callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, + save_solution, amr_callback, stepsize_callback) diff --git a/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl b/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl index 1f9aa3449b0..a47d94b62d5 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl @@ -71,6 +71,14 @@ analysis_callback = AnalysisCallback(semi, interval = analysis_interval, alive_callback = AliveCallback(analysis_interval = analysis_interval) +save_restart = SaveRestartCallback(interval = 100, + save_final_restart = true) + +save_solution = SaveSolutionCallback(interval = 100, + save_initial_solution = true, + save_final_solution = true, + solution_variables = cons2prim) + amr_controller = ControllerThreeLevel(semi, IndicatorMax(semi, variable = first), base_level = 1, med_level = 2, med_threshold = 0.1, @@ -85,6 +93,8 @@ stepsize_callback = StepsizeCallback(cfl = 1.2) callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, + save_restart, + save_solution, amr_callback, stepsize_callback) diff --git a/examples/t8code_3d_dgsem/elixir_advection_basic.jl b/examples/t8code_3d_dgsem/elixir_advection_basic.jl index f49462035aa..ae97a73d182 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_basic.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_basic.jl @@ -40,11 +40,19 @@ summary_callback = SummaryCallback() # The AnalysisCallback allows to analyse the solution in regular intervals and prints the results analysis_callback = AnalysisCallback(semi, interval = 100) +# The SaveRestartCallback allows to save a file from which a Trixi.jl simulation can be restarted +save_restart = SaveRestartCallback(interval = 100, + save_final_restart = true) + +# The SaveSolutionCallback allows to save the solution to a file in regular intervals +save_solution = SaveSolutionCallback(interval = 100, + solution_variables = cons2prim) + # The StepsizeCallback handles the re-calculation of the maximum Δt after each time step stepsize_callback = StepsizeCallback(cfl = 1.2) # Create a CallbackSet to collect all callbacks such that they can be passed to the ODE solver -callbacks = CallbackSet(summary_callback, analysis_callback, +callbacks = CallbackSet(summary_callback, analysis_callback, save_restart, save_solution, stepsize_callback) ############################################################################### diff --git a/examples/t8code_3d_dgsem/elixir_advection_nonconforming.jl b/examples/t8code_3d_dgsem/elixir_advection_nonconforming.jl index 8d7a48370f5..0755a76ef45 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_nonconforming.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_nonconforming.jl @@ -66,11 +66,15 @@ summary_callback = SummaryCallback() # The AnalysisCallback allows to analyse the solution in regular intervals and prints the results analysis_callback = AnalysisCallback(semi, interval = 100) +# The SaveSolutionCallback allows to save the solution to a file in regular intervals +save_solution = SaveSolutionCallback(interval = 100, + solution_variables = cons2prim) + # The StepsizeCallback handles the re-calculation of the maximum Δt after each time step stepsize_callback = StepsizeCallback(cfl = 1.6) # Create a CallbackSet to collect all callbacks such that they can be passed to the ODE solver -callbacks = CallbackSet(summary_callback, analysis_callback, +callbacks = CallbackSet(summary_callback, analysis_callback, save_solution, stepsize_callback) ############################################################################### diff --git a/examples/t8code_3d_dgsem/elixir_advection_restart.jl b/examples/t8code_3d_dgsem/elixir_advection_restart.jl new file mode 100644 index 00000000000..b3dead42399 --- /dev/null +++ b/examples/t8code_3d_dgsem/elixir_advection_restart.jl @@ -0,0 +1,41 @@ + +using OrdinaryDiffEq +using Trixi + +############################################################################### +# create a restart file + +trixi_include(@__MODULE__, joinpath(@__DIR__, "elixir_advection_basic.jl"), + trees_per_dimension = (2, 2, 2)) + +############################################################################### +# adapt the parameters that have changed compared to "elixir_advection_extended.jl" + +# Note: If you get a restart file from somewhere else, you need to provide +# appropriate setups in the elixir loading a restart file + +restart_filename = joinpath("out", "restart_000010.h5") +mesh = load_mesh(restart_filename) + +semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition_convergence_test, + solver) + +tspan = (load_time(restart_filename), 2.0) +dt = load_dt(restart_filename) +ode = semidiscretize(semi, tspan, restart_filename); + +# Do not overwrite the initial snapshot written by elixir_advection_extended.jl. +save_solution.condition.save_initial_solution = false + +integrator = init(ode, CarpenterKennedy2N54(williamson_condition = false), + dt = dt, # solve needs some value here but it will be overwritten by the stepsize_callback + save_everystep = false, callback = callbacks, maxiters = 100_000); + +# Get the last time index and work with that. +load_timestep!(integrator, restart_filename) + +############################################################################### +# run the simulation + +sol = solve!(integrator) +summary_callback() # print the timer summary diff --git a/examples/t8code_3d_dgsem/elixir_advection_unstructured_curved.jl b/examples/t8code_3d_dgsem/elixir_advection_unstructured_curved.jl index df1dbce2af1..f49137e054d 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_unstructured_curved.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_unstructured_curved.jl @@ -69,17 +69,21 @@ summary_callback = SummaryCallback() analysis_interval = 100 analysis_callback = AnalysisCallback(semi, interval = analysis_interval) +alive_callback = AliveCallback(analysis_interval = analysis_interval) + +# The SaveRestartCallback allows to save a file from which a Trixi.jl simulation can be restarted +save_restart = SaveRestartCallback(interval = 100, + save_final_restart = true) + # The SaveSolutionCallback allows to save the solution to a file in regular intervals save_solution = SaveSolutionCallback(interval = 100, solution_variables = cons2prim) -alive_callback = AliveCallback(analysis_interval = analysis_interval) - # The StepsizeCallback handles the re-calculation of the maximum Δt after each time step stepsize_callback = StepsizeCallback(cfl = 1.2) # Create a CallbackSet to collect all callbacks such that they can be passed to the ODE solver -callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, +callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, save_restart, save_solution, stepsize_callback) ############################################################################### diff --git a/examples/t8code_3d_dgsem/elixir_euler_ec.jl b/examples/t8code_3d_dgsem/elixir_euler_ec.jl index e1e4d850a86..e75b0f69636 100644 --- a/examples/t8code_3d_dgsem/elixir_euler_ec.jl +++ b/examples/t8code_3d_dgsem/elixir_euler_ec.jl @@ -68,11 +68,16 @@ analysis_callback = AnalysisCallback(semi, interval = analysis_interval) alive_callback = AliveCallback(analysis_interval = analysis_interval) +save_solution = SaveSolutionCallback(interval = 100, + save_initial_solution = true, + save_final_solution = true) + stepsize_callback = StepsizeCallback(cfl = 1.0) callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, + save_solution, stepsize_callback) ############################################################################### diff --git a/examples/t8code_3d_dgsem/elixir_euler_free_stream.jl b/examples/t8code_3d_dgsem/elixir_euler_free_stream.jl index 882e3aebebe..d45de658cc0 100644 --- a/examples/t8code_3d_dgsem/elixir_euler_free_stream.jl +++ b/examples/t8code_3d_dgsem/elixir_euler_free_stream.jl @@ -95,10 +95,16 @@ analysis_callback = AnalysisCallback(semi, interval = analysis_interval) alive_callback = AliveCallback(analysis_interval = analysis_interval) +save_solution = SaveSolutionCallback(interval = 100, + save_initial_solution = true, + save_final_solution = true, + solution_variables = cons2prim) + stepsize_callback = StepsizeCallback(cfl = 1.2) callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, + save_solution, stepsize_callback) ############################################################################### diff --git a/examples/t8code_3d_dgsem/elixir_euler_free_stream_extruded.jl b/examples/t8code_3d_dgsem/elixir_euler_free_stream_extruded.jl index 777cccf7ad7..d24512a4cdd 100644 --- a/examples/t8code_3d_dgsem/elixir_euler_free_stream_extruded.jl +++ b/examples/t8code_3d_dgsem/elixir_euler_free_stream_extruded.jl @@ -83,10 +83,16 @@ analysis_callback = AnalysisCallback(semi, interval = analysis_interval) alive_callback = AliveCallback(analysis_interval = analysis_interval) +save_solution = SaveSolutionCallback(interval = 100, + save_initial_solution = true, + save_final_solution = true, + solution_variables = cons2prim) + stepsize_callback = StepsizeCallback(cfl = 1.2) callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, + save_solution, stepsize_callback) ############################################################################### diff --git a/examples/t8code_3d_dgsem/elixir_euler_sedov.jl b/examples/t8code_3d_dgsem/elixir_euler_sedov.jl index 618b170b661..f897249ed2e 100644 --- a/examples/t8code_3d_dgsem/elixir_euler_sedov.jl +++ b/examples/t8code_3d_dgsem/elixir_euler_sedov.jl @@ -81,11 +81,16 @@ analysis_callback = AnalysisCallback(semi, interval = analysis_interval) alive_callback = AliveCallback(analysis_interval = analysis_interval) +save_solution = SaveSolutionCallback(interval = 100, + save_initial_solution = true, + save_final_solution = true) + stepsize_callback = StepsizeCallback(cfl = 0.5) callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, + save_solution, stepsize_callback) ############################################################################### diff --git a/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_curved.jl b/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_curved.jl index a06e7927dd0..4b87b646df9 100644 --- a/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_curved.jl +++ b/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_curved.jl @@ -97,10 +97,16 @@ analysis_callback = AnalysisCallback(semi, interval = analysis_interval) alive_callback = AliveCallback(analysis_interval = analysis_interval) +save_solution = SaveSolutionCallback(interval = 100, + save_initial_solution = true, + save_final_solution = true, + solution_variables = cons2prim) + stepsize_callback = StepsizeCallback(cfl = 0.6) callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, + save_solution, stepsize_callback); ############################################################################### diff --git a/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonperiodic.jl b/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonperiodic.jl index 7cb03bb312d..ce1662c8e50 100644 --- a/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonperiodic.jl +++ b/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonperiodic.jl @@ -47,10 +47,16 @@ analysis_callback = AnalysisCallback(semi, interval = analysis_interval) alive_callback = AliveCallback(analysis_interval = analysis_interval) +save_solution = SaveSolutionCallback(interval = 100, + save_initial_solution = true, + save_final_solution = true, + solution_variables = cons2prim) + stepsize_callback = StepsizeCallback(cfl = 0.6) callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, + save_solution, stepsize_callback) ############################################################################### diff --git a/src/auxiliary/t8code.jl b/src/auxiliary/t8code.jl index 83ef471e1c6..4c961e80008 100644 --- a/src/auxiliary/t8code.jl +++ b/src/auxiliary/t8code.jl @@ -17,8 +17,7 @@ function init_t8code() # Initialize the sc library, has to happen before we initialize t8code. let catch_signals = 0, print_backtrace = 0, log_handler = C_NULL T8code.Libt8.sc_init(mpi_comm(), catch_signals, print_backtrace, log_handler, - T8code.Libt8.SC_LP_DEBUG) - # T8code.Libt8.SC_LP_ERROR) + T8code.Libt8.SC_LP_ERROR) end if T8code.Libt8.p4est_is_initialized() == 0 @@ -27,8 +26,7 @@ function init_t8code() end # Initialize t8code with log level ERROR to prevent a lot of output in AMR simulations. - # t8_init(T8code.Libt8.SC_LP_ERROR) - t8_init(T8code.Libt8.SC_LP_DEBUG) + t8_init(T8code.Libt8.SC_LP_ERROR) if haskey(ENV, "TRIXI_T8CODE_SC_FINALIZE") # Normally, `sc_finalize` should always be called during shutdown of an diff --git a/src/callbacks_step/save_restart_dg.jl b/src/callbacks_step/save_restart_dg.jl index b83402c5f86..210d15d3982 100644 --- a/src/callbacks_step/save_restart_dg.jl +++ b/src/callbacks_step/save_restart_dg.jl @@ -48,7 +48,7 @@ function save_restart_file(u, time, dt, timestep, end function load_restart_file(mesh::Union{SerialTreeMesh, StructuredMesh, - UnstructuredMesh2D, SerialP4estMesh}, + UnstructuredMesh2D, SerialP4estMesh, SerialT8codeMesh}, equations, dg::DG, cache, restart_file) # allocate memory @@ -88,7 +88,7 @@ function load_restart_file(mesh::Union{SerialTreeMesh, StructuredMesh, end function save_restart_file(u, time, dt, timestep, - mesh::Union{ParallelTreeMesh, ParallelP4estMesh}, equations, + mesh::Union{ParallelTreeMesh, ParallelP4estMesh, ParallelT8codeMesh}, equations, dg::DG, cache, restart_callback) @unpack output_directory = restart_callback @@ -105,7 +105,7 @@ function save_restart_file(u, time, dt, timestep, end function save_restart_file_parallel(u, time, dt, timestep, - mesh::Union{ParallelTreeMesh, ParallelP4estMesh}, + mesh::Union{ParallelTreeMesh, ParallelP4estMesh, ParallelT8codeMesh}, equations, dg::DG, cache, filename) @@ -151,7 +151,7 @@ function save_restart_file_parallel(u, time, dt, timestep, end function save_restart_file_on_root(u, time, dt, timestep, - mesh::Union{ParallelTreeMesh, ParallelP4estMesh}, + mesh::Union{ParallelTreeMesh, ParallelP4estMesh, ParallelT8codeMesh}, equations, dg::DG, cache, filename) @@ -204,7 +204,7 @@ function save_restart_file_on_root(u, time, dt, timestep, return filename end -function load_restart_file(mesh::Union{ParallelTreeMesh, ParallelP4estMesh}, equations, +function load_restart_file(mesh::Union{ParallelTreeMesh, ParallelP4estMesh, ParallelT8codeMesh}, equations, dg::DG, cache, restart_file) if HDF5.has_parallel() load_restart_file_parallel(mesh, equations, dg, cache, restart_file) @@ -213,7 +213,7 @@ function load_restart_file(mesh::Union{ParallelTreeMesh, ParallelP4estMesh}, equ end end -function load_restart_file_parallel(mesh::Union{ParallelTreeMesh, ParallelP4estMesh}, +function load_restart_file_parallel(mesh::Union{ParallelTreeMesh, ParallelP4estMesh, ParallelT8codeMesh}, equations, dg::DG, cache, restart_file) # Calculate element and node counts by MPI rank @@ -264,7 +264,7 @@ function load_restart_file_parallel(mesh::Union{ParallelTreeMesh, ParallelP4estM return u_ode end -function load_restart_file_on_root(mesh::Union{ParallelTreeMesh, ParallelP4estMesh}, +function load_restart_file_on_root(mesh::Union{ParallelTreeMesh, ParallelP4estMesh, ParallelT8codeMesh}, equations, dg::DG, cache, restart_file) # Calculate element and node counts by MPI rank diff --git a/test/test_mpi_t8code_2d.jl b/test/test_mpi_t8code_2d.jl index 7c7fc03898c..c3a6ed7a253 100644 --- a/test/test_mpi_t8code_2d.jl +++ b/test/test_mpi_t8code_2d.jl @@ -111,6 +111,24 @@ const EXAMPLES_DIR = pkgdir(Trixi, "examples", "t8code_2d_dgsem") end end + @trixi_testset "elixir_advection_restart.jl" begin + @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_restart.jl"), + l2=[4.507575525876275e-6], + linf=[6.21489667023134e-5], + # With the default `maxiters = 1` in coverage tests, + # there would be no time steps after the restart. + coverage_override=(maxiters = 100_000,)) + + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end + end + @trixi_testset "elixir_euler_source_terms_nonconforming_unstructured_flag.jl" begin @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_euler_source_terms_nonconforming_unstructured_flag.jl"), diff --git a/test/test_mpi_t8code_3d.jl b/test/test_mpi_t8code_3d.jl index a15690a7629..a0fd3868d45 100644 --- a/test/test_mpi_t8code_3d.jl +++ b/test/test_mpi_t8code_3d.jl @@ -87,6 +87,24 @@ const EXAMPLES_DIR = pkgdir(Trixi, "examples", "t8code_3d_dgsem") end end + @trixi_testset "elixir_advection_restart.jl" begin + @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_restart.jl"), + l2=[0.002590388934758452], + linf=[0.01840757696885409], + # With the default `maxiters = 1` in coverage tests, + # there would be no time steps after the restart. + coverage_override=(maxiters = 100_000,)) + + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end + end + # Compressible Euler @trixi_testset "elixir_euler_source_terms_nonconforming_unstructured_curved.jl" begin @test_trixi_include(joinpath(EXAMPLES_DIR, diff --git a/test/test_t8code_2d.jl b/test/test_t8code_2d.jl index b63d2a105ac..f0607dd977d 100644 --- a/test/test_t8code_2d.jl +++ b/test/test_t8code_2d.jl @@ -15,20 +15,20 @@ mkdir(outdir) @testset "T8codeMesh2D" begin #! format: noindent -@trixi_testset "test save_mesh_file" begin - @test_throws Exception begin - # Save mesh file support will be added in the future. The following - # lines of code are here for satisfying code coverage. - - # Create dummy mesh. - mesh = T8codeMesh((1, 1), polydeg = 1, - mapping = Trixi.coordinates2mapping((-1.0, -1.0), (1.0, 1.0)), - initial_refinement_level = 1) - - # This call throws an error. - Trixi.save_mesh_file(mesh, "dummy") - end -end +# @trixi_testset "test save_mesh_file" begin +# @test_throws Exception begin +# # Save mesh file support will be added in the future. The following +# # lines of code are here for satisfying code coverage. +# +# # Create dummy mesh. +# mesh = T8codeMesh((1, 1), polydeg = 1, +# mapping = Trixi.coordinates2mapping((-1.0, -1.0), (1.0, 1.0)), +# initial_refinement_level = 1) +# +# # This call throws an error. +# Trixi.save_mesh_file(mesh, "dummy") +# end +# end @trixi_testset "test load mesh from path" begin mktempdir() do path @@ -152,6 +152,24 @@ end end end +@trixi_testset "elixir_advection_restart_amr.jl" begin + # This test is identical to the one in `test_p4est_2d.jl`. + @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_restart_amr.jl"), + l2=[2.869137983727866e-6], + linf=[3.8353423270964804e-5], + # With the default `maxiters = 1` in coverage tests, + # there would be no time steps after the restart. + coverage_override=(maxiters = 25,)) + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end +end + @trixi_testset "elixir_euler_source_terms_nonconforming_unstructured_flag.jl" begin # This test is identical to the one in `test_p4est_2d.jl`. @test_trixi_include(joinpath(EXAMPLES_DIR, diff --git a/test/test_t8code_3d.jl b/test/test_t8code_3d.jl index 81d2a7cdd85..e5adeeef8f5 100644 --- a/test/test_t8code_3d.jl +++ b/test/test_t8code_3d.jl @@ -110,6 +110,24 @@ mkdir(outdir) end end + # This test is identical to the one in `test_p4est_3d.jl`. + @trixi_testset "elixir_advection_restart.jl" begin + @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_restart.jl"), + l2=[0.002590388934758452], + linf=[0.01840757696885409], + # With the default `maxiters = 1` in coverage tests, + # there would be no time steps after the restart. + coverage_override=(maxiters = 100_000,)) + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end + end + # This test is identical to the one in `test_p4est_3d.jl`. @trixi_testset "elixir_euler_source_terms_nonconforming_unstructured_curved.jl" begin @test_trixi_include(joinpath(EXAMPLES_DIR, From 2f96869b64e15f7434a459b08cbefbe9f4028336 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Thu, 20 Jun 2024 14:57:43 +0200 Subject: [PATCH 51/81] Applied formatter. --- .../elixir_advection_extended.jl | 4 ++-- .../elixir_advection_nonconforming_flag.jl | 3 ++- src/auxiliary/t8code.jl | 2 +- src/callbacks_step/save_restart_dg.jl | 21 ++++++++++++------- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/examples/t8code_2d_dgsem/elixir_advection_extended.jl b/examples/t8code_2d_dgsem/elixir_advection_extended.jl index 44892faf40d..f7e06dd517e 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_extended.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_extended.jl @@ -28,8 +28,8 @@ trees_per_dimension = (19, 37) # Create curved mesh with 19 x 37 elements mesh = T8codeMesh(trees_per_dimension, polydeg = 3, - coordinates_min = coordinates_min, coordinates_max = coordinates_max, - periodicity = false) + coordinates_min = coordinates_min, coordinates_max = coordinates_max, + periodicity = false) # A semidiscretization collects data structures and functions for the spatial discretization semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition, solver, diff --git a/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl b/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl index f080b640f0d..7230c8c0b9e 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl @@ -77,7 +77,8 @@ save_solution = SaveSolutionCallback(interval = 100, stepsize_callback = StepsizeCallback(cfl = 1.6) # Create a CallbackSet to collect all callbacks such that they can be passed to the ODE solver -callbacks = CallbackSet(summary_callback, analysis_callback, save_solution, stepsize_callback) +callbacks = CallbackSet(summary_callback, analysis_callback, save_solution, + stepsize_callback) ############################################################################### # run the simulation diff --git a/src/auxiliary/t8code.jl b/src/auxiliary/t8code.jl index 4c961e80008..d7703b8bbb4 100644 --- a/src/auxiliary/t8code.jl +++ b/src/auxiliary/t8code.jl @@ -17,7 +17,7 @@ function init_t8code() # Initialize the sc library, has to happen before we initialize t8code. let catch_signals = 0, print_backtrace = 0, log_handler = C_NULL T8code.Libt8.sc_init(mpi_comm(), catch_signals, print_backtrace, log_handler, - T8code.Libt8.SC_LP_ERROR) + T8code.Libt8.SC_LP_ERROR) end if T8code.Libt8.p4est_is_initialized() == 0 diff --git a/src/callbacks_step/save_restart_dg.jl b/src/callbacks_step/save_restart_dg.jl index 210d15d3982..c5bf2191c7a 100644 --- a/src/callbacks_step/save_restart_dg.jl +++ b/src/callbacks_step/save_restart_dg.jl @@ -48,7 +48,8 @@ function save_restart_file(u, time, dt, timestep, end function load_restart_file(mesh::Union{SerialTreeMesh, StructuredMesh, - UnstructuredMesh2D, SerialP4estMesh, SerialT8codeMesh}, + UnstructuredMesh2D, SerialP4estMesh, + SerialT8codeMesh}, equations, dg::DG, cache, restart_file) # allocate memory @@ -88,7 +89,8 @@ function load_restart_file(mesh::Union{SerialTreeMesh, StructuredMesh, end function save_restart_file(u, time, dt, timestep, - mesh::Union{ParallelTreeMesh, ParallelP4estMesh, ParallelT8codeMesh}, equations, + mesh::Union{ParallelTreeMesh, ParallelP4estMesh, + ParallelT8codeMesh}, equations, dg::DG, cache, restart_callback) @unpack output_directory = restart_callback @@ -105,7 +107,8 @@ function save_restart_file(u, time, dt, timestep, end function save_restart_file_parallel(u, time, dt, timestep, - mesh::Union{ParallelTreeMesh, ParallelP4estMesh, ParallelT8codeMesh}, + mesh::Union{ParallelTreeMesh, ParallelP4estMesh, + ParallelT8codeMesh}, equations, dg::DG, cache, filename) @@ -151,7 +154,8 @@ function save_restart_file_parallel(u, time, dt, timestep, end function save_restart_file_on_root(u, time, dt, timestep, - mesh::Union{ParallelTreeMesh, ParallelP4estMesh, ParallelT8codeMesh}, + mesh::Union{ParallelTreeMesh, ParallelP4estMesh, + ParallelT8codeMesh}, equations, dg::DG, cache, filename) @@ -204,7 +208,8 @@ function save_restart_file_on_root(u, time, dt, timestep, return filename end -function load_restart_file(mesh::Union{ParallelTreeMesh, ParallelP4estMesh, ParallelT8codeMesh}, equations, +function load_restart_file(mesh::Union{ParallelTreeMesh, ParallelP4estMesh, + ParallelT8codeMesh}, equations, dg::DG, cache, restart_file) if HDF5.has_parallel() load_restart_file_parallel(mesh, equations, dg, cache, restart_file) @@ -213,7 +218,8 @@ function load_restart_file(mesh::Union{ParallelTreeMesh, ParallelP4estMesh, Para end end -function load_restart_file_parallel(mesh::Union{ParallelTreeMesh, ParallelP4estMesh, ParallelT8codeMesh}, +function load_restart_file_parallel(mesh::Union{ParallelTreeMesh, ParallelP4estMesh, + ParallelT8codeMesh}, equations, dg::DG, cache, restart_file) # Calculate element and node counts by MPI rank @@ -264,7 +270,8 @@ function load_restart_file_parallel(mesh::Union{ParallelTreeMesh, ParallelP4estM return u_ode end -function load_restart_file_on_root(mesh::Union{ParallelTreeMesh, ParallelP4estMesh, ParallelT8codeMesh}, +function load_restart_file_on_root(mesh::Union{ParallelTreeMesh, ParallelP4estMesh, + ParallelT8codeMesh}, equations, dg::DG, cache, restart_file) # Calculate element and node counts by MPI rank From df2bdf41b443e2864db835b8623af38df345c1a3 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Fri, 21 Jun 2024 11:17:55 +0200 Subject: [PATCH 52/81] Minor adjustments. --- src/callbacks_step/amr.jl | 2 ++ src/meshes/t8code_mesh.jl | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/callbacks_step/amr.jl b/src/callbacks_step/amr.jl index b0afd02aff8..8696ccd0d31 100644 --- a/src/callbacks_step/amr.jl +++ b/src/callbacks_step/amr.jl @@ -788,6 +788,8 @@ function (amr_callback::AMRCallback)(u_ode::AbstractVector, mesh::T8codeMesh, reinitialize_boundaries!(semi.boundary_conditions, cache) end + mesh.unsaved_changes |= has_changed + # Return true if there were any cells coarsened or refined, otherwise false. return has_changed end diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index d75741ca995..317b3a7d9b8 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -1045,7 +1045,7 @@ function count_interfaces(forest::Ptr{t8_forest}, ndims) elseif level < neighbor_level local_num_mpi_mortars += 1 - global_mortar_id = 2 * ndims(mesh) * current_linear_id + iface + global_mortar_id = 2 * ndims * current_linear_id + iface else # level > neighbor_level neighbor_global_ghost_itree = ghost_global_treeids[findlast(ghost_tree_element_offsets .<= @@ -1055,7 +1055,7 @@ function count_interfaces(forest::Ptr{t8_forest}, ndims) t8_element_get_linear_id(neighbor_scheme, neighbor_leaves[1], max_level) - global_mortar_id = 2 * ndims(mesh) * neighbor_linear_id + + global_mortar_id = 2 * ndims * neighbor_linear_id + dual_faces[1] if !(global_mortar_id in visited_global_mortar_ids) @@ -1455,7 +1455,7 @@ function get_cmesh_info(cmesh::Ptr{t8_cmesh}, ndims) # Allocate arrays. treeIDs = zeros(Int, num_interfaces) neighIDs = zeros(Int, num_interfaces) - orientations = zeros(Int32, num_interfaces) + orientations = zeros(Int8, num_interfaces) faces = zeros(Int8, num_interfaces) duals = zeros(Int8, num_interfaces) From 7ee23b60e8281ff4a26ccb9ee11c820e91f1d546 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Mon, 24 Jun 2024 14:28:39 +0200 Subject: [PATCH 53/81] Code refinement. Enabled partitioning after mesh loading. --- src/meshes/mesh_io.jl | 29 +++++++++++++++++++---- src/meshes/t8code_mesh.jl | 50 ++++++++++++++++++++++----------------- 2 files changed, 52 insertions(+), 27 deletions(-) diff --git a/src/meshes/mesh_io.jl b/src/meshes/mesh_io.jl index 8958936f771..49a99b516e9 100644 --- a/src/meshes/mesh_io.jl +++ b/src/meshes/mesh_io.jl @@ -225,6 +225,11 @@ function save_mesh_file(mesh::P4estMesh, output_directory, timestep, mpi_paralle return filename end +# This routine works for both, serial and MPI parallel mode. The forest +# information is collected on all ranks and then gathered by the root rank. +# Since only the `levels` array of unsigned bytes is bascially independent per +# rank it is not worth the effort to have a collective write to the HDF5 file. +# Instead, `levels` gets gathered by the root rank and written in serial. function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, mpi_parallel::Any) @@ -239,12 +244,25 @@ function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, end # Retrieve refinement levels of all elements. - levels = get_levels(mesh) + local_levels = get_levels(mesh) if mpi_isparallel() - levels = MPI.Gather(levels, mpi_root(), mpi_comm()) + count = [length(local_levels)] + counts = MPI.Gather(view(count, 1), mpi_root(), mpi_comm()) + + if mpi_isroot() + levels = similar(local_levels, ncellsglobal(mesh)) + MPI.Gatherv!(local_levels, MPI.VBuffer(levels, counts), + mpi_root(), mpi_comm()) + else + MPI.Gatherv!(local_levels, nothing, mpi_root(), mpi_comm()) + end + else + levels = local_levels end - # Retrieve number of elements per tree. + # Retrieve the number of elements per tree. Since a tree can be distributed + # among multiple ranks a reduction operation sums them all up. The latter + # is done on the root rank only. num_global_trees = t8_forest_get_num_global_trees(mesh.forest) num_elements_per_tree = zeros(t8_gloidx_t, num_global_trees) num_local_trees = t8_forest_get_num_local_trees(mesh.forest) @@ -256,7 +274,7 @@ function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, end if mpi_isparallel() - num_elements_per_tree = MPI.Reduce!(num_elements_per_tree, +, mpi_comm()) + MPI.Reduce!(num_elements_per_tree, +, mpi_comm()) end # Since the mesh attributes are replicated on all ranks, only save from MPI @@ -274,7 +292,7 @@ function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, attributes(file)["mesh_type"] = get_name(mesh) attributes(file)["ndims"] = ndims(mesh) attributes(file)["ntrees"] = ntrees(mesh) - attributes(file)["nelements"] = ncells(mesh) + attributes(file)["nelements"] = ncellsglobal(mesh) file["tree_node_coordinates"] = mesh.tree_node_coordinates file["nodes"] = Vector(mesh.nodes) @@ -495,6 +513,7 @@ function load_mesh_parallel(mesh_file::AbstractString; n_cells_max, RealT) mesh = P4estMesh{ndims_}(p4est, tree_node_coordinates, nodes, boundary_names, mesh_file, false, true) + elseif mesh_type == "T8codeMesh" if mpi_isroot() ndims, ntrees, nelements, tree_node_coordinates, nodes, diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 317b3a7d9b8..10b9060422a 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -208,32 +208,33 @@ function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, cum_sum_num_elements_per_tree = cumsum(num_elements_per_tree) - # Compute the offset within the to-be-reconstructed forest depending on the + global_element_id = 0 # zero-based index + + # Compute the offset within the to-be-reconstructed forest. Depends on the # MPI rank resp. first global tree id. - virtual_element_index = 1 # one-based index if mpi_rank() > 0 last_global_tree_id_of_preceding_rank = t8_forest_global_tree_id(forest, 0) - 1 - virtual_element_index += cum_sum_num_elements_per_tree[last_global_tree_id_of_preceding_rank + 1] + global_element_id += cum_sum_num_elements_per_tree[last_global_tree_id_of_preceding_rank + 1] end function adapt_callback(forest, local_tree_id, eclass_scheme, local_element_id, elements, is_family, user_data) - # Check if we are already in the next tree in terms of the `virtual_element_index`. + # Check if we are already in the next tree in terms of the `global_element_id`. global_tree_id = t8_forest_global_tree_id(forest, local_tree_id) - if virtual_element_index > cum_sum_num_elements_per_tree[global_tree_id + 1] + if global_element_id + 1 > cum_sum_num_elements_per_tree[global_tree_id + 1] return 0 end # Test if we already reached the targeted level. level = t8_element_level(eclass_scheme, elements[1]) - if level < levels[virtual_element_index] + if level < levels[global_element_id + 1] return 1 # Go one refinement level deeper. end # Targeted level is reached. - virtual_element_index += 1 + global_element_id += 1 return 0 end @@ -243,9 +244,9 @@ function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, forest = adapt(forest, adapt_callback; recursive = true, balance = false, partition = false, ghost = false, user_data = C_NULL) - # if mpi_isparallel() - # forest = t8_forest_partition(forest) - # end + if mpi_isparallel() + forest = partition(forest) + end return T8codeMesh{ndims}(forest, tree_node_coordinates, nodes, boundary_names, "") end @@ -878,7 +879,9 @@ function adapt(forest::Ptr{t8_forest}, adapt_callback; recursive = true, balance t8_forest_set_partition(new_forest, set_from, set_for_coarsening) end - t8_forest_set_ghost(new_forest, ghost, T8_GHOST_FACES) # Note: MPI support not available yet so it is a dummy call. + if ghost + t8_forest_set_ghost(new_forest, ghost, T8_GHOST_FACES) + end # The old forest is destroyed here. # Call `t8_forest_ref(Ref(mesh.forest))` to keep it. @@ -914,27 +917,30 @@ function balance!(mesh::T8codeMesh) return nothing end -""" - Trixi.partition!(mesh::T8codeMesh) - -Partition a `T8codeMesh` in order to redistribute elements evenly among MPI ranks. - -# Arguments -- `mesh::T8codeMesh`: Initialized mesh object. -""" -function partition!(mesh::T8codeMesh) +function partition(forest::Ptr{t8_forest}) new_forest_ref = Ref{t8_forest_t}() t8_forest_init(new_forest_ref) new_forest = new_forest_ref[] - let set_from = mesh.forest, do_ghost = 1, allow_for_coarsening = 1 + let set_from = forest, do_ghost = 1, allow_for_coarsening = 1 t8_forest_set_partition(new_forest, set_from, allow_for_coarsening) t8_forest_set_ghost(new_forest, do_ghost, T8_GHOST_FACES) t8_forest_commit(new_forest) end - mesh.forest = new_forest + return new_forest +end + +""" + Trixi.partition!(mesh::T8codeMesh) +Partition a `T8codeMesh` in order to redistribute elements evenly among MPI ranks. + +# Arguments +- `mesh::T8codeMesh`: Initialized mesh object. +""" +function partition!(mesh::T8codeMesh) + mesh.forest = partition(mesh.forest) return nothing end From 0404f3b4b632398cd343af0fa8929949f20d1ed6 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Mon, 24 Jun 2024 14:38:36 +0200 Subject: [PATCH 54/81] Applied formatter and fixed typos. --- src/meshes/mesh_io.jl | 2 +- src/meshes/t8code_mesh.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/meshes/mesh_io.jl b/src/meshes/mesh_io.jl index 49a99b516e9..00242ba0169 100644 --- a/src/meshes/mesh_io.jl +++ b/src/meshes/mesh_io.jl @@ -227,7 +227,7 @@ end # This routine works for both, serial and MPI parallel mode. The forest # information is collected on all ranks and then gathered by the root rank. -# Since only the `levels` array of unsigned bytes is bascially independent per +# Since only the `levels` array of unsigned bytes is basicially independent per # rank it is not worth the effort to have a collective write to the HDF5 file. # Instead, `levels` gets gathered by the root rank and written in serial. function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 10b9060422a..a7f949706d4 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -245,7 +245,7 @@ function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, partition = false, ghost = false, user_data = C_NULL) if mpi_isparallel() - forest = partition(forest) + forest = partition(forest) end return T8codeMesh{ndims}(forest, tree_node_coordinates, nodes, boundary_names, "") From 99972ccd8656d0b7b24a4c80d6ce7b3a64e85e7f Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Mon, 24 Jun 2024 14:45:33 +0200 Subject: [PATCH 55/81] Removed commented out section. --- test/test_t8code_2d.jl | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/test/test_t8code_2d.jl b/test/test_t8code_2d.jl index f0607dd977d..279e858e8e6 100644 --- a/test/test_t8code_2d.jl +++ b/test/test_t8code_2d.jl @@ -15,21 +15,6 @@ mkdir(outdir) @testset "T8codeMesh2D" begin #! format: noindent -# @trixi_testset "test save_mesh_file" begin -# @test_throws Exception begin -# # Save mesh file support will be added in the future. The following -# # lines of code are here for satisfying code coverage. -# -# # Create dummy mesh. -# mesh = T8codeMesh((1, 1), polydeg = 1, -# mapping = Trixi.coordinates2mapping((-1.0, -1.0), (1.0, 1.0)), -# initial_refinement_level = 1) -# -# # This call throws an error. -# Trixi.save_mesh_file(mesh, "dummy") -# end -# end - @trixi_testset "test load mesh from path" begin mktempdir() do path @test_throws "Unknown file extension: .unknown_ext" begin From f59322c91b162878951b2c8ee37a9542a951c935 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Mon, 24 Jun 2024 15:16:29 +0200 Subject: [PATCH 56/81] Added missing union type member. --- src/callbacks_step/save_solution_dg.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/callbacks_step/save_solution_dg.jl b/src/callbacks_step/save_solution_dg.jl index 33e4997ca93..50da208ccef 100644 --- a/src/callbacks_step/save_solution_dg.jl +++ b/src/callbacks_step/save_solution_dg.jl @@ -200,7 +200,8 @@ function save_solution_file_parallel(data, time, dt, timestep, n_vars, end function save_solution_file_on_root(data, time, dt, timestep, n_vars, - mesh::Union{ParallelTreeMesh, ParallelP4estMesh}, + mesh::Union{ParallelTreeMesh, ParallelP4estMesh, + ParallelT8codeMesh}, equations, dg::DG, cache, solution_variables, filename, element_variables = Dict{Symbol, Any}()) From be2fcdd7892d19d3e422c1245a0505ea4aa09918 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Wed, 26 Jun 2024 13:57:28 +0200 Subject: [PATCH 57/81] Switching from UInt64 to UInt128 in global interface/mortar id computation. --- src/meshes/t8code_mesh.jl | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index a7f949706d4..ec524fc2a4a 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -1,3 +1,5 @@ +using Printf + """ T8codeMesh{NDIMS} <: AbstractMesh{NDIMS} @@ -972,10 +974,10 @@ function count_interfaces(forest::Ptr{t8_forest}, ndims) local_num_mpi_conform = 0 local_num_mpi_mortars = 0 - visited_global_mortar_ids = Set{UInt64}([]) + visited_global_mortar_ids = Set{UInt128}([]) max_level = t8_forest_get_maxlevel(forest) #UInt64 - max_tree_num_elements = UInt64(2^ndims)^max_level + max_tree_num_elements = UInt128(2^ndims)^max_level if mpi_isparallel() ghost_num_trees = t8_forest_ghost_num_trees(forest) @@ -1144,12 +1146,12 @@ function fill_mesh_info!(mesh::T8codeMesh, interfaces, mortars, boundaries, # Helper variables to compute unique global MPI interface/mortar ids. max_level = t8_forest_get_maxlevel(mesh.forest) #UInt64 - max_tree_num_elements = UInt64(2^ndims(mesh))^max_level + max_tree_num_elements = (UInt128(2)^ndims(mesh))^max_level # These two variables help to ensure that we count MPI mortars from smaller # elements point of view only once. - visited_global_mortar_ids = Set{UInt64}([]) - global_mortar_id_to_local = Dict{UInt64, Int}([]) + visited_global_mortar_ids = Set{UInt128}([]) + global_mortar_id_to_local = Dict{UInt128, Int}([]) cmesh = t8_forest_get_cmesh(mesh.forest) From 0283428954309fb140c67b87544e31a298f4ceb9 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Wed, 26 Jun 2024 13:58:18 +0200 Subject: [PATCH 58/81] Switching from UInt64 to UInt128 in global interface/mortar id computation (II). --- src/solvers/dgsem_t8code/containers_parallel.jl | 4 ++-- src/solvers/dgsem_t8code/dg_parallel.jl | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/solvers/dgsem_t8code/containers_parallel.jl b/src/solvers/dgsem_t8code/containers_parallel.jl index 0cb3f5887a0..381f672f43d 100644 --- a/src/solvers/dgsem_t8code/containers_parallel.jl +++ b/src/solvers/dgsem_t8code/containers_parallel.jl @@ -21,8 +21,8 @@ function reinitialize_containers!(mesh::ParallelT8codeMesh, equations, dg::DGSEM mpi_interfaces = mpi_interfaces, # Temporary arrays for updating `mpi_cache`. - global_mortar_ids = fill(UInt64(0), nmpimortars(mpi_mortars)), - global_interface_ids = fill(UInt64(0), nmpiinterfaces(mpi_interfaces)), + global_mortar_ids = fill(UInt128(0), nmpimortars(mpi_mortars)), + global_interface_ids = fill(UInt128(0), nmpiinterfaces(mpi_interfaces)), neighbor_ranks_mortar = Vector{Vector{Int}}(undef, nmpimortars(mpi_mortars)), neighbor_ranks_interface = fill(-1, nmpiinterfaces(mpi_interfaces))) diff --git a/src/solvers/dgsem_t8code/dg_parallel.jl b/src/solvers/dgsem_t8code/dg_parallel.jl index ece614b7d75..d33d5445610 100644 --- a/src/solvers/dgsem_t8code/dg_parallel.jl +++ b/src/solvers/dgsem_t8code/dg_parallel.jl @@ -25,8 +25,8 @@ function create_cache(mesh::ParallelT8codeMesh, equations::AbstractEquations, dg mpi_mesh_info = (mpi_mortars = mpi_mortars, mpi_interfaces = mpi_interfaces, - global_mortar_ids = fill(UInt64(0), nmpimortars(mpi_mortars)), - global_interface_ids = fill(UInt64(0), + global_mortar_ids = fill(UInt128(0), nmpimortars(mpi_mortars)), + global_interface_ids = fill(UInt128(0), nmpiinterfaces(mpi_interfaces)), neighbor_ranks_mortar = Vector{Vector{Int}}(undef, nmpimortars(mpi_mortars)), @@ -75,7 +75,6 @@ function init_mpi_cache!(mpi_cache::P4estMPICache, mesh::ParallelT8codeMesh, nvars, nnodes, uEltype) - n_elements_global = Int(t8_forest_get_global_num_elements(mesh.forest)) n_elements_local = Int(t8_forest_get_local_num_elements(mesh.forest)) From db9d4c63cdaa3aabc35156d2537bd53f022bb7a0 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Wed, 26 Jun 2024 13:59:01 +0200 Subject: [PATCH 59/81] Adding more tests. --- test/test_mpi_t8code_2d.jl | 10 +++++----- test/test_t8code_2d.jl | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/test/test_mpi_t8code_2d.jl b/test/test_mpi_t8code_2d.jl index c3a6ed7a253..db8e3025846 100644 --- a/test/test_mpi_t8code_2d.jl +++ b/test/test_mpi_t8code_2d.jl @@ -80,9 +80,9 @@ const EXAMPLES_DIR = pkgdir(Trixi, "examples", "t8code_2d_dgsem") @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_amr_solution_independent.jl"), # Expected errors are exactly the same as with TreeMesh! - l2=[4.933027431215839e-5], - linf=[0.00048678461161243136], - coverage_override=(maxiters = 6,)) + l2=[4.949660644033807e-5], + linf=[0.0004867846262313763], + coverage_override=(maxiters = 6,), atol=1e-9) # Ensure that we do not have excessive memory allocations # (e.g., from type instabilities) @@ -97,8 +97,8 @@ const EXAMPLES_DIR = pkgdir(Trixi, "examples", "t8code_2d_dgsem") @trixi_testset "elixir_advection_amr_unstructured_flag.jl" begin @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_amr_unstructured_flag.jl"), - l2=[0.001980652042312077], - linf=[0.0328882442132265], + l2=[0.001993165013217687], + linf=[0.032891018571625796], coverage_override=(maxiters = 6,)) # Ensure that we do not have excessive memory allocations diff --git a/test/test_t8code_2d.jl b/test/test_t8code_2d.jl index 279e858e8e6..418e3d422f1 100644 --- a/test/test_t8code_2d.jl +++ b/test/test_t8code_2d.jl @@ -137,6 +137,24 @@ end end end +@trixi_testset "elixir_advection_restart.jl" begin + @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_restart.jl"), + l2=[4.507575525876275e-6], + linf=[6.21489667023134e-5], + # With the default `maxiters = 1` in coverage tests, + # there would be no time steps after the restart. + coverage_override=(maxiters = 100_000,)) + + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end +end + @trixi_testset "elixir_advection_restart_amr.jl" begin # This test is identical to the one in `test_p4est_2d.jl`. @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_restart_amr.jl"), From b6e133391c554afe2c95b43720831b51326bc806 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Wed, 26 Jun 2024 14:13:18 +0200 Subject: [PATCH 60/81] Applied formatter. --- .../t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl | 3 ++- src/solvers/dgsem_t8code/containers_parallel.jl | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl b/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl index 352455d986f..ca7b1da7fe2 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl @@ -69,7 +69,8 @@ amr_controller = ControllerThreeLevel(semi, IndicatorMax(semi, variable = first) amr_callback = AMRCallback(semi, amr_controller, interval = 5, adapt_initial_condition = true, - adapt_initial_condition_only_refine = true) + adapt_initial_condition_only_refine = true, + dynamic_load_balancing = false) stepsize_callback = StepsizeCallback(cfl = 0.7) diff --git a/src/solvers/dgsem_t8code/containers_parallel.jl b/src/solvers/dgsem_t8code/containers_parallel.jl index 381f672f43d..731f6aea7e2 100644 --- a/src/solvers/dgsem_t8code/containers_parallel.jl +++ b/src/solvers/dgsem_t8code/containers_parallel.jl @@ -22,7 +22,8 @@ function reinitialize_containers!(mesh::ParallelT8codeMesh, equations, dg::DGSEM # Temporary arrays for updating `mpi_cache`. global_mortar_ids = fill(UInt128(0), nmpimortars(mpi_mortars)), - global_interface_ids = fill(UInt128(0), nmpiinterfaces(mpi_interfaces)), + global_interface_ids = fill(UInt128(0), + nmpiinterfaces(mpi_interfaces)), neighbor_ranks_mortar = Vector{Vector{Int}}(undef, nmpimortars(mpi_mortars)), neighbor_ranks_interface = fill(-1, nmpiinterfaces(mpi_interfaces))) From 1aea5cdcc84012be0365f68a4109b426e94401db Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Wed, 26 Jun 2024 14:23:53 +0200 Subject: [PATCH 61/81] Removed Printf. --- src/meshes/t8code_mesh.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index ec524fc2a4a..de36659a7cc 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -1,5 +1,3 @@ -using Printf - """ T8codeMesh{NDIMS} <: AbstractMesh{NDIMS} From b60382337302c0ad65e27ab5ee2e32c2d418b7d2 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Fri, 28 Jun 2024 14:14:34 +0200 Subject: [PATCH 62/81] Incorporated review comments and code polish. --- ...ixir_advection_amr_solution_independent.jl | 9 ++- .../elixir_advection_amr_unstructured_flag.jl | 7 +++ .../elixir_advection_restart_amr.jl | 10 +++- examples/t8code_2d_dgsem/elixir_mhd_rotor.jl | 9 ++- .../t8code_3d_dgsem/elixir_advection_amr.jl | 9 ++- ...lixir_advection_amr_unstructured_curved.jl | 9 ++- src/meshes/mesh_io.jl | 9 +-- src/meshes/t8code_mesh.jl | 55 ++++++++++++------- 8 files changed, 88 insertions(+), 29 deletions(-) diff --git a/examples/t8code_2d_dgsem/elixir_advection_amr_solution_independent.jl b/examples/t8code_2d_dgsem/elixir_advection_amr_solution_independent.jl index d10cb167cb0..ab7d19a3207 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_amr_solution_independent.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_amr_solution_independent.jl @@ -129,7 +129,14 @@ amr_controller = ControllerThreeLevel(semi, amr_callback = AMRCallback(semi, amr_controller, interval = 5, adapt_initial_condition = true, - adapt_initial_condition_only_refine = true) + adapt_initial_condition_only_refine = true, + dynamic_load_balancing = false) +# We disable `dynamic_load_balancing` for now, since t8code does not support +# partioning for coarsening yet. That is, a complete family of elements always +# stays on rank and is not split up due to partioning. Without this feature +# dynamic AMR simulations are not pefectly deterministic regarding to +# convergent tests. Once this feature is available in t8code load balancing is +# enabled again. stepsize_callback = StepsizeCallback(cfl = 1.6) diff --git a/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl b/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl index ca7b1da7fe2..2c2c55fb5b8 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl @@ -71,6 +71,13 @@ amr_callback = AMRCallback(semi, amr_controller, adapt_initial_condition = true, adapt_initial_condition_only_refine = true, dynamic_load_balancing = false) +# We disable `dynamic_load_balancing` for now, since t8code does not support +# partioning for coarsening yet. That is, a complete family of elements always +# stays on rank and is not split up due to partioning. Without this feature +# dynamic AMR simulations are not pefectly deterministic regarding to +# convergent tests. Once this feature is available in t8code load balancing is +# enabled again. + stepsize_callback = StepsizeCallback(cfl = 0.7) diff --git a/examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl b/examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl index fd3623dd88b..2703eeedb0c 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl @@ -37,7 +37,15 @@ amr_controller = ControllerThreeLevel(semi, IndicatorMax(semi, variable = first) amr_callback = AMRCallback(semi, amr_controller, interval = 5, adapt_initial_condition = true, - adapt_initial_condition_only_refine = true) + adapt_initial_condition_only_refine = true, + dynamic_load_balancing = false) +# We disable `dynamic_load_balancing` for now, since t8code does not support +# partioning for coarsening yet. That is, a complete family of elements always +# stays on rank and is not split up due to partioning. Without this feature +# dynamic AMR simulations are not pefectly deterministic regarding to +# convergent tests. Once this feature is available in t8code load balancing is +# enabled again. + callbacks_ext = CallbackSet(amr_callback, callbacks.discrete_callbacks...) integrator = init(ode, CarpenterKennedy2N54(williamson_condition = false), diff --git a/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl b/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl index a09ea46fa34..87f46696096 100644 --- a/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl +++ b/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl @@ -108,7 +108,14 @@ amr_controller = ControllerThreeLevel(semi, amr_indicator, amr_callback = AMRCallback(semi, amr_controller, interval = 5, adapt_initial_condition = true, - adapt_initial_condition_only_refine = true) + adapt_initial_condition_only_refine = true, + dynamic_load_balancing = false) +# We disable `dynamic_load_balancing` for now, since t8code does not support +# partioning for coarsening yet. That is, a complete family of elements always +# stays on rank and is not split up due to partioning. Without this feature +# dynamic AMR simulations are not pefectly deterministic regarding to +# convergent tests. Once this feature is available in t8code load balancing is +# enabled again. cfl = 0.5 stepsize_callback = StepsizeCallback(cfl = cfl) diff --git a/examples/t8code_3d_dgsem/elixir_advection_amr.jl b/examples/t8code_3d_dgsem/elixir_advection_amr.jl index 9ecc8955383..48e8a57cbcf 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_amr.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_amr.jl @@ -52,7 +52,14 @@ amr_controller = ControllerThreeLevel(semi, IndicatorMax(semi, variable = first) amr_callback = AMRCallback(semi, amr_controller, interval = 5, adapt_initial_condition = true, - adapt_initial_condition_only_refine = true) + adapt_initial_condition_only_refine = true, + dynamic_load_balancing = false) +# We disable `dynamic_load_balancing` for now, since t8code does not support +# partioning for coarsening yet. That is, a complete family of elements always +# stays on rank and is not split up due to partioning. Without this feature +# dynamic AMR simulations are not pefectly deterministic regarding to +# convergent tests. Once this feature is available in t8code load balancing is +# enabled again. stepsize_callback = StepsizeCallback(cfl = 1.2) diff --git a/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl b/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl index a47d94b62d5..e65312b2f29 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl @@ -86,7 +86,14 @@ amr_controller = ControllerThreeLevel(semi, IndicatorMax(semi, variable = first) amr_callback = AMRCallback(semi, amr_controller, interval = 5, adapt_initial_condition = true, - adapt_initial_condition_only_refine = true) + adapt_initial_condition_only_refine = true, + dynamic_load_balancing = false) +# We disable `dynamic_load_balancing` for now, since t8code does not support +# partioning for coarsening yet. That is, a complete family of elements always +# stays on rank and is not split up due to partioning. Without this feature +# dynamic AMR simulations are not pefectly deterministic regarding to +# convergent tests. Once this feature is available in t8code load balancing is +# enabled again. stepsize_callback = StepsizeCallback(cfl = 1.2) diff --git a/src/meshes/mesh_io.jl b/src/meshes/mesh_io.jl index 00242ba0169..8c0f7754402 100644 --- a/src/meshes/mesh_io.jl +++ b/src/meshes/mesh_io.jl @@ -227,11 +227,12 @@ end # This routine works for both, serial and MPI parallel mode. The forest # information is collected on all ranks and then gathered by the root rank. -# Since only the `levels` array of unsigned bytes is basicially independent per -# rank it is not worth the effort to have a collective write to the HDF5 file. -# Instead, `levels` gets gathered by the root rank and written in serial. +# Since only the `levels` array of UInt8 and the global number of elements per +# tree (Int32) is necessary to reconstruct the forest it is not worth the +# effort to have a collective write to the HDF5 file. Instead, `levels` and +# `num_elements_per_tree` gets gathered by the root rank and written to disk. function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, - mpi_parallel::Any) + mpi_parallel::Union{False,True}) # Create output directory (if it does not exist). mpi_isroot() && mkpath(output_directory) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index de36659a7cc..b5c0a2519d3 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -105,31 +105,42 @@ function Base.show(io::IO, ::MIME"text/plain", mesh::T8codeMesh) end """ - T8codeMesh{NDIMS, RealT}(forest, boundary_names; polydeg = 1, mapping = nothing) + T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, + boundary_names, treeIDs, neighIDs, faces, duals, + orientations, levels, num_elements_per_tree)) -Main mesh constructor for the `T8codeMesh` wrapping around a given t8code -`forest` object. This constructor is typically called by other `T8codeMesh` -constructors. +Constructor for the `T8codeMesh`. Typically called by the `load_mesh` routine. # Arguments -- `forest`: Pointer to a t8code forest. +- `ndims`: Dimension of the mesh. +- `ntrees`: Global number of trees. +- `nelements`: Global number of elements. +- `tree_node_coordinates`: Node coordinates for each tree: [dimension, i, j, k, tree] +- `nodes`: Array of interpolation nodes. - `boundary_names`: List of boundary names. -- `polydeg::Integer`: Polynomial degree used to store the geometry of the mesh. - The mapping will be approximated by an interpolation polynomial - of the specified degree for each tree. -- `mapping`: A function of `NDIMS` variables to describe the mapping that transforms - the imported mesh to the physical domain. Use `nothing` for the identity map. +- `treeIDs`: List of tree IDs. The lenght is the number of conforming interfaces of the coarse mesh. +- `neighIDs`: List of neighboring tree IDs. Same length as `treeIDs`. +- `faces`: List of face IDs. Same length as `treeIDs`. +- `duals`: List of face IDs of the neighboring tree. Same length as `treeIDs`. +- `orientations`: Orientation number of the interface. Same length as `treeIDs`. +- `levels`: List of levels of each element. Has length `nelements`. +- `num_elements_per_tree`: List of global number of elements per tree. Has length `ntrees`. + +Returns a `T8codeMesh` object with a forest reconstructed by the input arguments. """ function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, - boundary_names, elemIDs, neighIDs, faces, duals, + boundary_names, treeIDs, neighIDs, faces, duals, orientations, levels, num_elements_per_tree) Trixi.cmesh_ref = Ref(t8_cmesh_t()) t8_cmesh_init(Trixi.cmesh_ref) cmesh = Trixi.cmesh_ref[] # Use linear geometry for now. There is no real Lagrange geometry - # implementation yet in t8code. - Trixi.linear_geom = Trixi.t8_geometry_linear_new(2) + # implementation (volume nodes) yet in t8code. Moreover, we need to store + # the pointer variables in the `Trixi` package in order to avoid garbage + # collection. Otherwise t8code segfaults. This is an un-feature of t8code + # (recently introduced) and will be fixed in the near future. + Trixi.linear_geom = Trixi.t8_geometry_linear_new(ndims) Trixi.linear_geom_ptr = pointer_from_objref(Ref(Trixi.linear_geom)) t8_cmesh_register_geometry(cmesh, Trixi.linear_geom_ptr) @@ -188,12 +199,12 @@ function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, vertices[24] = tree_node_coordinates[3, N, N, N, i] end - t8_cmesh_set_tree_vertices(cmesh, i - 1, vertices, 3) + t8_cmesh_set_tree_vertices(cmesh, i - 1, vertices, 2^ndims) end # Connect the coarse mesh elements. - for i in 1:length(elemIDs) - t8_cmesh_set_join(cmesh, elemIDs[i], neighIDs[i], faces[i], duals[i], + for i in eachindex(treeIDs) + t8_cmesh_set_join(cmesh, treeIDs[i], neighIDs[i], faces[i], duals[i], orientations[i]) end @@ -212,7 +223,7 @@ function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, # Compute the offset within the to-be-reconstructed forest. Depends on the # MPI rank resp. first global tree id. - if mpi_rank() > 0 + if mpi_rank() > 0 && t8_forest_get_local_num_elements(forest) > 0 last_global_tree_id_of_preceding_rank = t8_forest_global_tree_id(forest, 0) - 1 global_element_id += cum_sum_num_elements_per_tree[last_global_tree_id_of_preceding_rank + 1] end @@ -244,6 +255,8 @@ function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, forest = adapt(forest, adapt_callback; recursive = true, balance = false, partition = false, ghost = false, user_data = C_NULL) + @assert t8_forest_get_global_num_elements(forest) == nelements + if mpi_isparallel() forest = partition(forest) end @@ -974,7 +987,7 @@ function count_interfaces(forest::Ptr{t8_forest}, ndims) visited_global_mortar_ids = Set{UInt128}([]) - max_level = t8_forest_get_maxlevel(forest) #UInt64 + max_level = t8_forest_get_maxlevel(forest) max_tree_num_elements = UInt128(2^ndims)^max_level if mpi_isparallel() @@ -1143,8 +1156,8 @@ function fill_mesh_info!(mesh::T8codeMesh, interfaces, mortars, boundaries, ] # Helper variables to compute unique global MPI interface/mortar ids. - max_level = t8_forest_get_maxlevel(mesh.forest) #UInt64 - max_tree_num_elements = (UInt128(2)^ndims(mesh))^max_level + max_level = t8_forest_get_maxlevel(mesh.forest) + max_tree_num_elements = UInt128(2^ndims(mesh))^max_level # These two variables help to ensure that we count MPI mortars from smaller # elements point of view only once. @@ -1438,6 +1451,8 @@ function get_cmesh_info(mesh::T8codeMesh) return get_cmesh_info(cmesh, ndims(mesh)) end +# Note, `cmesh` is not partitioned as of now. +# Every MPI rank has a full copy of the `cmesh`. function get_cmesh_info(cmesh::Ptr{t8_cmesh}, ndims) num_trees = t8_cmesh_get_num_trees(cmesh) num_faces = 2 * ndims From 6ef17a4d3ee645e94927c4675625a181bb614422 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Fri, 28 Jun 2024 14:15:13 +0200 Subject: [PATCH 63/81] Applied formatter. --- .../t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl | 1 - src/meshes/mesh_io.jl | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl b/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl index 2c2c55fb5b8..88739660d2d 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl @@ -78,7 +78,6 @@ amr_callback = AMRCallback(semi, amr_controller, # convergent tests. Once this feature is available in t8code load balancing is # enabled again. - stepsize_callback = StepsizeCallback(cfl = 0.7) callbacks = CallbackSet(summary_callback, diff --git a/src/meshes/mesh_io.jl b/src/meshes/mesh_io.jl index 8c0f7754402..0fc20775f7a 100644 --- a/src/meshes/mesh_io.jl +++ b/src/meshes/mesh_io.jl @@ -232,7 +232,7 @@ end # effort to have a collective write to the HDF5 file. Instead, `levels` and # `num_elements_per_tree` gets gathered by the root rank and written to disk. function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, - mpi_parallel::Union{False,True}) + mpi_parallel::Union{False, True}) # Create output directory (if it does not exist). mpi_isroot() && mkpath(output_directory) From 69b7ac10baf50a67ea447fc47198e09101745b36 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Fri, 28 Jun 2024 15:19:07 +0200 Subject: [PATCH 64/81] Fixed typos. --- .../elixir_advection_amr_solution_independent.jl | 4 ++-- .../t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl | 4 ++-- examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl | 4 ++-- examples/t8code_2d_dgsem/elixir_mhd_rotor.jl | 4 ++-- examples/t8code_3d_dgsem/elixir_advection_amr.jl | 4 ++-- .../elixir_advection_amr_unstructured_curved.jl | 4 ++-- src/meshes/t8code_mesh.jl | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/examples/t8code_2d_dgsem/elixir_advection_amr_solution_independent.jl b/examples/t8code_2d_dgsem/elixir_advection_amr_solution_independent.jl index ab7d19a3207..cfa0a34bb34 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_amr_solution_independent.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_amr_solution_independent.jl @@ -132,8 +132,8 @@ amr_callback = AMRCallback(semi, amr_controller, adapt_initial_condition_only_refine = true, dynamic_load_balancing = false) # We disable `dynamic_load_balancing` for now, since t8code does not support -# partioning for coarsening yet. That is, a complete family of elements always -# stays on rank and is not split up due to partioning. Without this feature +# partitioning for coarsening yet. That is, a complete family of elements always +# stays on rank and is not split up due to partitioning. Without this feature # dynamic AMR simulations are not pefectly deterministic regarding to # convergent tests. Once this feature is available in t8code load balancing is # enabled again. diff --git a/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl b/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl index 88739660d2d..bd82809e016 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl @@ -72,8 +72,8 @@ amr_callback = AMRCallback(semi, amr_controller, adapt_initial_condition_only_refine = true, dynamic_load_balancing = false) # We disable `dynamic_load_balancing` for now, since t8code does not support -# partioning for coarsening yet. That is, a complete family of elements always -# stays on rank and is not split up due to partioning. Without this feature +# partitioning for coarsening yet. That is, a complete family of elements always +# stays on rank and is not split up due to partitioning. Without this feature # dynamic AMR simulations are not pefectly deterministic regarding to # convergent tests. Once this feature is available in t8code load balancing is # enabled again. diff --git a/examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl b/examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl index 2703eeedb0c..68fa726227b 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl @@ -40,8 +40,8 @@ amr_callback = AMRCallback(semi, amr_controller, adapt_initial_condition_only_refine = true, dynamic_load_balancing = false) # We disable `dynamic_load_balancing` for now, since t8code does not support -# partioning for coarsening yet. That is, a complete family of elements always -# stays on rank and is not split up due to partioning. Without this feature +# partitioning for coarsening yet. That is, a complete family of elements always +# stays on rank and is not split up due to partitioning. Without this feature # dynamic AMR simulations are not pefectly deterministic regarding to # convergent tests. Once this feature is available in t8code load balancing is # enabled again. diff --git a/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl b/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl index 87f46696096..88a2888de62 100644 --- a/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl +++ b/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl @@ -111,8 +111,8 @@ amr_callback = AMRCallback(semi, amr_controller, adapt_initial_condition_only_refine = true, dynamic_load_balancing = false) # We disable `dynamic_load_balancing` for now, since t8code does not support -# partioning for coarsening yet. That is, a complete family of elements always -# stays on rank and is not split up due to partioning. Without this feature +# partitioning for coarsening yet. That is, a complete family of elements always +# stays on rank and is not split up due to partitioning. Without this feature # dynamic AMR simulations are not pefectly deterministic regarding to # convergent tests. Once this feature is available in t8code load balancing is # enabled again. diff --git a/examples/t8code_3d_dgsem/elixir_advection_amr.jl b/examples/t8code_3d_dgsem/elixir_advection_amr.jl index 48e8a57cbcf..d2a0e3fb666 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_amr.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_amr.jl @@ -55,8 +55,8 @@ amr_callback = AMRCallback(semi, amr_controller, adapt_initial_condition_only_refine = true, dynamic_load_balancing = false) # We disable `dynamic_load_balancing` for now, since t8code does not support -# partioning for coarsening yet. That is, a complete family of elements always -# stays on rank and is not split up due to partioning. Without this feature +# partitioning for coarsening yet. That is, a complete family of elements always +# stays on rank and is not split up due to partitioning. Without this feature # dynamic AMR simulations are not pefectly deterministic regarding to # convergent tests. Once this feature is available in t8code load balancing is # enabled again. diff --git a/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl b/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl index e65312b2f29..56b2574849f 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl @@ -89,8 +89,8 @@ amr_callback = AMRCallback(semi, amr_controller, adapt_initial_condition_only_refine = true, dynamic_load_balancing = false) # We disable `dynamic_load_balancing` for now, since t8code does not support -# partioning for coarsening yet. That is, a complete family of elements always -# stays on rank and is not split up due to partioning. Without this feature +# partitioning for coarsening yet. That is, a complete family of elements always +# stays on rank and is not split up due to partitioning. Without this feature # dynamic AMR simulations are not pefectly deterministic regarding to # convergent tests. Once this feature is available in t8code load balancing is # enabled again. diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index b5c0a2519d3..19ee671f6ce 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -118,7 +118,7 @@ Constructor for the `T8codeMesh`. Typically called by the `load_mesh` routine. - `tree_node_coordinates`: Node coordinates for each tree: [dimension, i, j, k, tree] - `nodes`: Array of interpolation nodes. - `boundary_names`: List of boundary names. -- `treeIDs`: List of tree IDs. The lenght is the number of conforming interfaces of the coarse mesh. +- `treeIDs`: List of tree IDs. The length is the number of conforming interfaces of the coarse mesh. - `neighIDs`: List of neighboring tree IDs. Same length as `treeIDs`. - `faces`: List of face IDs. Same length as `treeIDs`. - `duals`: List of face IDs of the neighboring tree. Same length as `treeIDs`. From 3bbd3a15538240977dcdef1b5c6127b24a1c5650 Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Tue, 2 Jul 2024 17:07:08 +0200 Subject: [PATCH 65/81] Update examples/t8code_2d_dgsem/elixir_advection_restart.jl Co-authored-by: Joshua Lampert <51029046+JoshuaLampert@users.noreply.github.com> --- examples/t8code_2d_dgsem/elixir_advection_restart.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/t8code_2d_dgsem/elixir_advection_restart.jl b/examples/t8code_2d_dgsem/elixir_advection_restart.jl index 0f573714c1f..6fb14c60038 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_restart.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_restart.jl @@ -6,7 +6,7 @@ using Trixi # create a restart file elixir_file = "elixir_advection_extended.jl" -restart_file = "restart_000021.h5" +restart_file = "restart_000000021.h5" trixi_include(@__MODULE__, joinpath(@__DIR__, elixir_file)) From ab9c33f020e8160b8fb136216a01a6fd95e96214 Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Tue, 2 Jul 2024 17:07:16 +0200 Subject: [PATCH 66/81] Update examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl Co-authored-by: Joshua Lampert <51029046+JoshuaLampert@users.noreply.github.com> --- examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl b/examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl index 68fa726227b..2c0f14c8d10 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl @@ -6,7 +6,7 @@ using Trixi # create a restart file elixir_file = "elixir_advection_extended.jl" -restart_file = "restart_000021.h5" +restart_file = "restart_000000021.h5" trixi_include(@__MODULE__, joinpath(@__DIR__, elixir_file)) From 9e49219a248f131178b051097acd5fb085e0a909 Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Tue, 2 Jul 2024 17:07:27 +0200 Subject: [PATCH 67/81] Update examples/t8code_3d_dgsem/elixir_advection_restart.jl Co-authored-by: Joshua Lampert <51029046+JoshuaLampert@users.noreply.github.com> --- examples/t8code_3d_dgsem/elixir_advection_restart.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/t8code_3d_dgsem/elixir_advection_restart.jl b/examples/t8code_3d_dgsem/elixir_advection_restart.jl index b3dead42399..9d19d81cf47 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_restart.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_restart.jl @@ -14,7 +14,7 @@ trixi_include(@__MODULE__, joinpath(@__DIR__, "elixir_advection_basic.jl"), # Note: If you get a restart file from somewhere else, you need to provide # appropriate setups in the elixir loading a restart file -restart_filename = joinpath("out", "restart_000010.h5") +restart_filename = joinpath("out", "restart_000000010.h5") mesh = load_mesh(restart_filename) semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition_convergence_test, From 677236f1970be823380fb3d47bf7f99a4af244d5 Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Tue, 2 Jul 2024 17:07:38 +0200 Subject: [PATCH 68/81] Update src/meshes/mesh_io.jl Co-authored-by: Joshua Lampert <51029046+JoshuaLampert@users.noreply.github.com> --- src/meshes/mesh_io.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meshes/mesh_io.jl b/src/meshes/mesh_io.jl index 684d8c34943..a93401b624f 100644 --- a/src/meshes/mesh_io.jl +++ b/src/meshes/mesh_io.jl @@ -239,7 +239,7 @@ function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, # Determine file name based on existence of meaningful time step. if timestep > 0 - filename = joinpath(output_directory, @sprintf("mesh_%06d.h5", timestep)) + filename = joinpath(output_directory, @sprintf("mesh_%09d.h5", timestep)) else filename = joinpath(output_directory, "mesh.h5") end From 404fc85d5716a45c8aaca3d907d92cfd4df0954a Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Thu, 4 Jul 2024 10:51:33 +0200 Subject: [PATCH 69/81] Update examples/t8code_3d_dgsem/elixir_advection_restart.jl Co-authored-by: Joshua Lampert <51029046+JoshuaLampert@users.noreply.github.com> --- examples/t8code_3d_dgsem/elixir_advection_restart.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/t8code_3d_dgsem/elixir_advection_restart.jl b/examples/t8code_3d_dgsem/elixir_advection_restart.jl index 9d19d81cf47..dfce8d2558a 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_restart.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_restart.jl @@ -1,4 +1,3 @@ - using OrdinaryDiffEq using Trixi From 611a5761efbdce07a3c20cbc444e603a511c4f2e Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Thu, 4 Jul 2024 10:51:57 +0200 Subject: [PATCH 70/81] Update src/meshes/t8code_mesh.jl Co-authored-by: Joshua Lampert <51029046+JoshuaLampert@users.noreply.github.com> --- src/meshes/t8code_mesh.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 19ee671f6ce..95f1d2ccba4 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -107,7 +107,7 @@ end """ T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, boundary_names, treeIDs, neighIDs, faces, duals, - orientations, levels, num_elements_per_tree)) + orientations, levels, num_elements_per_tree) Constructor for the `T8codeMesh`. Typically called by the `load_mesh` routine. From 7161d1b7b57604c67e9ef2fd707286dfe82df618 Mon Sep 17 00:00:00 2001 From: Johannes Markert <10619309+jmark@users.noreply.github.com> Date: Thu, 4 Jul 2024 11:23:24 +0200 Subject: [PATCH 71/81] Update src/meshes/t8code_mesh.jl Co-authored-by: Joshua Lampert <51029046+JoshuaLampert@users.noreply.github.com> --- src/meshes/t8code_mesh.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 95f1d2ccba4..eda5c376434 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -1446,7 +1446,7 @@ function get_levels(mesh::T8codeMesh) end function get_cmesh_info(mesh::T8codeMesh) - @assert t8_forest_is_committed(mesh.forest) != 0 + @assert t8_forest_is_committed(mesh.forest) == 1 cmesh = t8_forest_get_cmesh(mesh.forest) return get_cmesh_info(cmesh, ndims(mesh)) end From 15cdc0b9bf975ed58c57088fa39eb9111723dc5a Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Fri, 5 Jul 2024 10:24:55 +0200 Subject: [PATCH 72/81] Removing last test in t8code 2D MPI to investigate problems in Github CI. --- test/test_mpi_t8code_2d.jl | 50 +++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/test/test_mpi_t8code_2d.jl b/test/test_mpi_t8code_2d.jl index db8e3025846..58dfd3c6fe3 100644 --- a/test/test_mpi_t8code_2d.jl +++ b/test/test_mpi_t8code_2d.jl @@ -129,31 +129,31 @@ const EXAMPLES_DIR = pkgdir(Trixi, "examples", "t8code_2d_dgsem") end end - @trixi_testset "elixir_euler_source_terms_nonconforming_unstructured_flag.jl" begin - @test_trixi_include(joinpath(EXAMPLES_DIR, - "elixir_euler_source_terms_nonconforming_unstructured_flag.jl"), - l2=[ - 0.0034516244508588046, - 0.0023420334036925493, - 0.0024261923964557187, - 0.004731710454271893, - ], - linf=[ - 0.04155789011775046, - 0.024772109862748914, - 0.03759938693042297, - 0.08039824959535657, - ]) - - # Ensure that we do not have excessive memory allocations - # (e.g., from type instabilities) - let - t = sol.t[end] - u_ode = sol.u[end] - du_ode = similar(u_ode) - @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 - end - end + # @trixi_testset "elixir_euler_source_terms_nonconforming_unstructured_flag.jl" begin + # @test_trixi_include(joinpath(EXAMPLES_DIR, + # "elixir_euler_source_terms_nonconforming_unstructured_flag.jl"), + # l2=[ + # 0.0034516244508588046, + # 0.0023420334036925493, + # 0.0024261923964557187, + # 0.004731710454271893, + # ], + # linf=[ + # 0.04155789011775046, + # 0.024772109862748914, + # 0.03759938693042297, + # 0.08039824959535657, + # ]) + + # # Ensure that we do not have excessive memory allocations + # # (e.g., from type instabilities) + # let + # t = sol.t[end] + # u_ode = sol.u[end] + # du_ode = similar(u_ode) + # @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + # end + # end end end # T8codeMesh MPI From 971d2cf08abf934722573934ebeba63aaa1c506f Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Mon, 8 Jul 2024 11:22:50 +0200 Subject: [PATCH 73/81] Refactored a bit. --- src/meshes/t8code_mesh.jl | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index eda5c376434..eb16122b660 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -26,11 +26,15 @@ mutable struct T8codeMesh{NDIMS, RealT <: Real, IsParallel, NDIMSP2, NNODES} <: nmpiinterfaces :: Int nmpimortars :: Int - unsaved_changes::Bool + unsaved_changes :: Bool + + # Keeps a reference to the geometry handler in order to avoid gargabe + # collection if necessary. + geometry :: Any function T8codeMesh{NDIMS}(forest::Ptr{t8_forest}, tree_node_coordinates, nodes, boundary_names, - current_filename) where {NDIMS} + current_filename; geometry = nothing) where {NDIMS} is_parallel = mpi_isparallel() ? True() : False() mesh = new{NDIMS, Float64, typeof(is_parallel), NDIMS + 2, length(nodes)}(forest, @@ -41,6 +45,7 @@ mutable struct T8codeMesh{NDIMS, RealT <: Real, IsParallel, NDIMSP2, NNODES} <: mesh.current_filename = current_filename mesh.tree_node_coordinates = tree_node_coordinates mesh.unsaved_changes = true + mesh.geometry = geometry finalizer(mesh) do mesh # When finalizing `mesh.forest`, `mesh.scheme` and `mesh.cmesh` are @@ -131,18 +136,19 @@ Returns a `T8codeMesh` object with a forest reconstructed by the input arguments function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, boundary_names, treeIDs, neighIDs, faces, duals, orientations, levels, num_elements_per_tree) - Trixi.cmesh_ref = Ref(t8_cmesh_t()) - t8_cmesh_init(Trixi.cmesh_ref) - cmesh = Trixi.cmesh_ref[] + # Initialize the `cmesh` object. + cmesh_ref = Ref(t8_cmesh_t()) + t8_cmesh_init(cmesh_ref) + cmesh = cmesh_ref[] # Use linear geometry for now. There is no real Lagrange geometry # implementation (volume nodes) yet in t8code. Moreover, we need to store - # the pointer variables in the `Trixi` package in order to avoid garbage + # the pointer variables in the `mesh` object in order to avoid garbage # collection. Otherwise t8code segfaults. This is an un-feature of t8code # (recently introduced) and will be fixed in the near future. - Trixi.linear_geom = Trixi.t8_geometry_linear_new(ndims) - Trixi.linear_geom_ptr = pointer_from_objref(Ref(Trixi.linear_geom)) - t8_cmesh_register_geometry(cmesh, Trixi.linear_geom_ptr) + linear_geom = t8_geometry_linear_new(ndims) + linear_geom_ptr = pointer_from_objref(Ref(linear_geom)) + t8_cmesh_register_geometry(cmesh, linear_geom_ptr) # Determine element class. eclass = ndims > 2 ? T8_ECLASS_HEX : T8_ECLASS_QUAD @@ -261,7 +267,7 @@ function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, forest = partition(forest) end - return T8codeMesh{ndims}(forest, tree_node_coordinates, nodes, boundary_names, "") + return T8codeMesh{ndims}(forest, tree_node_coordinates, nodes, boundary_names, ""; geometry = linear_geom_ptr) end """ From a3c169c34e30235ce4f7ba12aead27431499baa7 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Mon, 8 Jul 2024 11:27:03 +0200 Subject: [PATCH 74/81] Applied formatter. --- src/meshes/t8code_mesh.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index eb16122b660..ad226c8db6b 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -26,11 +26,11 @@ mutable struct T8codeMesh{NDIMS, RealT <: Real, IsParallel, NDIMSP2, NNODES} <: nmpiinterfaces :: Int nmpimortars :: Int - unsaved_changes :: Bool + unsaved_changes::Bool # Keeps a reference to the geometry handler in order to avoid gargabe # collection if necessary. - geometry :: Any + geometry::Any function T8codeMesh{NDIMS}(forest::Ptr{t8_forest}, tree_node_coordinates, nodes, boundary_names, @@ -267,7 +267,8 @@ function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, forest = partition(forest) end - return T8codeMesh{ndims}(forest, tree_node_coordinates, nodes, boundary_names, ""; geometry = linear_geom_ptr) + return T8codeMesh{ndims}(forest, tree_node_coordinates, nodes, boundary_names, ""; + geometry = linear_geom_ptr) end """ From 63835207e4c372ebadf7157c49152ed2b4344485 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Mon, 8 Jul 2024 11:27:48 +0200 Subject: [PATCH 75/81] Removed commented code. --- test/test_mpi_t8code_2d.jl | 50 +++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/test/test_mpi_t8code_2d.jl b/test/test_mpi_t8code_2d.jl index 58dfd3c6fe3..db8e3025846 100644 --- a/test/test_mpi_t8code_2d.jl +++ b/test/test_mpi_t8code_2d.jl @@ -129,31 +129,31 @@ const EXAMPLES_DIR = pkgdir(Trixi, "examples", "t8code_2d_dgsem") end end - # @trixi_testset "elixir_euler_source_terms_nonconforming_unstructured_flag.jl" begin - # @test_trixi_include(joinpath(EXAMPLES_DIR, - # "elixir_euler_source_terms_nonconforming_unstructured_flag.jl"), - # l2=[ - # 0.0034516244508588046, - # 0.0023420334036925493, - # 0.0024261923964557187, - # 0.004731710454271893, - # ], - # linf=[ - # 0.04155789011775046, - # 0.024772109862748914, - # 0.03759938693042297, - # 0.08039824959535657, - # ]) - - # # Ensure that we do not have excessive memory allocations - # # (e.g., from type instabilities) - # let - # t = sol.t[end] - # u_ode = sol.u[end] - # du_ode = similar(u_ode) - # @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 - # end - # end + @trixi_testset "elixir_euler_source_terms_nonconforming_unstructured_flag.jl" begin + @test_trixi_include(joinpath(EXAMPLES_DIR, + "elixir_euler_source_terms_nonconforming_unstructured_flag.jl"), + l2=[ + 0.0034516244508588046, + 0.0023420334036925493, + 0.0024261923964557187, + 0.004731710454271893, + ], + linf=[ + 0.04155789011775046, + 0.024772109862748914, + 0.03759938693042297, + 0.08039824959535657, + ]) + + # Ensure that we do not have excessive memory allocations + # (e.g., from type instabilities) + let + t = sol.t[end] + u_ode = sol.u[end] + du_ode = similar(u_ode) + @test (@allocated Trixi.rhs!(du_ode, u_ode, semi, t)) < 1000 + end + end end end # T8codeMesh MPI From 41ebc0762130c0bbe58f4bf9f572c1f986775150 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Tue, 9 Jul 2024 16:46:38 +0200 Subject: [PATCH 76/81] Added LOG_LEVEL variable. --- src/auxiliary/t8code.jl | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/auxiliary/t8code.jl b/src/auxiliary/t8code.jl index d7703b8bbb4..17b90c92ad1 100644 --- a/src/auxiliary/t8code.jl +++ b/src/auxiliary/t8code.jl @@ -14,19 +14,25 @@ function init_t8code() return nothing end - # Initialize the sc library, has to happen before we initialize t8code. - let catch_signals = 0, print_backtrace = 0, log_handler = C_NULL - T8code.Libt8.sc_init(mpi_comm(), catch_signals, print_backtrace, log_handler, - T8code.Libt8.SC_LP_ERROR) + # Initialize `libsc`, `p4est`, and `t8code` with log level + # `SC_LP_ERROR` to prevent a lot of output in AMR simulations + # For development, log level `SC_LP_DEBUG` is recommended. + LOG_LEVEL = T8code.Libt8.SC_LP_ERROR + + if T8code.Libt8.sc_is_initialized() == 0 + # Initialize the sc library, has to happen before we initialize t8code. + let catch_signals = 0, print_backtrace = 0, log_handler = C_NULL + T8code.Libt8.sc_init(mpi_comm(), catch_signals, print_backtrace, log_handler, + LOG_LEVEL) + end end if T8code.Libt8.p4est_is_initialized() == 0 - # Initialize `p4est` with log level ERROR to prevent a lot of output in AMR simulations - T8code.Libt8.p4est_init(C_NULL, T8code.Libt8.SC_LP_ERROR) + T8code.Libt8.p4est_init(C_NULL, LOG_LEVEL) end - # Initialize t8code with log level ERROR to prevent a lot of output in AMR simulations. - t8_init(T8code.Libt8.SC_LP_ERROR) + # Initialize t8code. + t8_init(LOG_LEVEL) if haskey(ENV, "TRIXI_T8CODE_SC_FINALIZE") # Normally, `sc_finalize` should always be called during shutdown of an From e268b33b920d6fac946d0ede046e412f5d48f9b6 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Tue, 9 Jul 2024 17:26:57 +0200 Subject: [PATCH 77/81] Added t8code interface simplfication and stitched memory leak. --- Project.toml | 2 +- src/meshes/t8code_mesh.jl | 43 ++++++++++++++++----------------------- 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/Project.toml b/Project.toml index 954b22ccaf1..995d46eba4d 100644 --- a/Project.toml +++ b/Project.toml @@ -99,7 +99,7 @@ StaticArrays = "1.5" StrideArrays = "0.1.26" StructArrays = "0.6.11" SummationByPartsOperators = "0.5.41" -T8code = "0.5" +T8code = "0.6" TimerOutputs = "0.5.7" Triangulate = "2.2" TriplotBase = "0.1" diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index ad226c8db6b..5089a33bba8 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -28,13 +28,9 @@ mutable struct T8codeMesh{NDIMS, RealT <: Real, IsParallel, NDIMSP2, NNODES} <: unsaved_changes::Bool - # Keeps a reference to the geometry handler in order to avoid gargabe - # collection if necessary. - geometry::Any - function T8codeMesh{NDIMS}(forest::Ptr{t8_forest}, tree_node_coordinates, nodes, boundary_names, - current_filename; geometry = nothing) where {NDIMS} + current_filename) where {NDIMS} is_parallel = mpi_isparallel() ? True() : False() mesh = new{NDIMS, Float64, typeof(is_parallel), NDIMS + 2, length(nodes)}(forest, @@ -45,7 +41,6 @@ mutable struct T8codeMesh{NDIMS, RealT <: Real, IsParallel, NDIMSP2, NNODES} <: mesh.current_filename = current_filename mesh.tree_node_coordinates = tree_node_coordinates mesh.unsaved_changes = true - mesh.geometry = geometry finalizer(mesh) do mesh # When finalizing `mesh.forest`, `mesh.scheme` and `mesh.cmesh` are @@ -60,6 +55,7 @@ mutable struct T8codeMesh{NDIMS, RealT <: Real, IsParallel, NDIMSP2, NNODES} <: # objects during long-running sessions. if !MPI.Finalized() t8_forest_unref(Ref(mesh.forest)) + mesh.forest = C_NULL end end @@ -136,19 +132,13 @@ Returns a `T8codeMesh` object with a forest reconstructed by the input arguments function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, boundary_names, treeIDs, neighIDs, faces, duals, orientations, levels, num_elements_per_tree) - # Initialize the `cmesh` object. - cmesh_ref = Ref(t8_cmesh_t()) - t8_cmesh_init(cmesh_ref) - cmesh = cmesh_ref[] + # Allocate new cmesh object. + cmesh = t8_cmesh_new() # Use linear geometry for now. There is no real Lagrange geometry - # implementation (volume nodes) yet in t8code. Moreover, we need to store - # the pointer variables in the `mesh` object in order to avoid garbage - # collection. Otherwise t8code segfaults. This is an un-feature of t8code - # (recently introduced) and will be fixed in the near future. + # implementation (volume nodes) yet in t8code. linear_geom = t8_geometry_linear_new(ndims) - linear_geom_ptr = pointer_from_objref(Ref(linear_geom)) - t8_cmesh_register_geometry(cmesh, linear_geom_ptr) + t8_cmesh_register_geometry(cmesh, linear_geom) # Determine element class. eclass = ndims > 2 ? T8_ECLASS_HEX : T8_ECLASS_QUAD @@ -267,8 +257,7 @@ function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, forest = partition(forest) end - return T8codeMesh{ndims}(forest, tree_node_coordinates, nodes, boundary_names, ""; - geometry = linear_geom_ptr) + return T8codeMesh{ndims}(forest, tree_node_coordinates, nodes, boundary_names, "") end """ @@ -1090,11 +1079,12 @@ function count_interfaces(forest::Ptr{t8_forest}, ndims) end end end - end - t8_free(dual_faces_ref[]) - t8_free(pneighbor_leaves_ref[]) - t8_free(pelement_indices_ref[]) + t8_element_destroy(neighbor_scheme, num_neighbors, neighbor_leaves) + t8_free(dual_faces_ref[]) + t8_free(pneighbor_leaves_ref[]) + t8_free(pelement_indices_ref[]) + end end # for current_index += 1 @@ -1434,11 +1424,12 @@ function fill_mesh_info!(mesh::T8codeMesh, interfaces, mortars, boundaries, end end end - end - t8_free(dual_faces_ref[]) - t8_free(pneighbor_leaves_ref[]) - t8_free(pelement_indices_ref[]) + t8_element_destroy(neighbor_scheme, num_neighbors, neighbor_leaves) + t8_free(dual_faces_ref[]) + t8_free(pneighbor_leaves_ref[]) + t8_free(pelement_indices_ref[]) + end # num_neighbors end # for iface current_index += 1 From 1e00cb6d9190b2546703638cbca57e4ff8a4d3ca Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Tue, 9 Jul 2024 17:34:56 +0200 Subject: [PATCH 78/81] Applied formatter. --- src/auxiliary/t8code.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/auxiliary/t8code.jl b/src/auxiliary/t8code.jl index 17b90c92ad1..86af91b5b51 100644 --- a/src/auxiliary/t8code.jl +++ b/src/auxiliary/t8code.jl @@ -22,7 +22,8 @@ function init_t8code() if T8code.Libt8.sc_is_initialized() == 0 # Initialize the sc library, has to happen before we initialize t8code. let catch_signals = 0, print_backtrace = 0, log_handler = C_NULL - T8code.Libt8.sc_init(mpi_comm(), catch_signals, print_backtrace, log_handler, + T8code.Libt8.sc_init(mpi_comm(), catch_signals, print_backtrace, + log_handler, LOG_LEVEL) end end From 74405d6bcdd7610a4a6d65c69e4f561202cc7b46 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Wed, 10 Jul 2024 13:46:19 +0200 Subject: [PATCH 79/81] 3D support. --- src/Trixi.jl | 3 +- src/callbacks_step/save_solution_dg.jl | 73 ++++- src/equations/equations.jl | 1 + src/equations/three_equations_3d.jl | 411 ++++++++++++++++++++++++ src/meshes/mesh_io.jl | 18 +- src/meshes/t8code_mesh.jl | 3 + src/solvers/dgsem_p4est/dg_3d.jl | 9 +- src/solvers/dgsem_tree/dg_2d.jl | 88 ++--- src/solvers/dgsem_tree/indicators_3d.jl | 37 +++ 9 files changed, 592 insertions(+), 51 deletions(-) create mode 100644 src/equations/three_equations_3d.jl diff --git a/src/Trixi.jl b/src/Trixi.jl index fe8bbf785d7..28d9e68e114 100644 --- a/src/Trixi.jl +++ b/src/Trixi.jl @@ -162,7 +162,7 @@ export AcousticPerturbationEquations2D, ShallowWaterEquationsQuasi1D, LinearizedEulerEquations1D, LinearizedEulerEquations2D, LinearizedEulerEquations3D, PolytropicEulerEquations2D, - ThreeEquations2D, + ThreeEquations2D, ThreeEquations3D, TrafficFlowLWREquations1D export LaplaceDiffusion1D, LaplaceDiffusion2D, LaplaceDiffusion3D, @@ -188,6 +188,7 @@ export flux, flux_central, flux_lax_friedrichs, flux_hll, flux_hllc, flux_hlle, FluxRotated, flux_shima_etal_turbo, flux_ranocha_turbo, FluxHydrostaticReconstruction, + flux_nonconservative_ThreeEquations, flux_nonconservative_ThreeEquations_well, FluxUpwind diff --git a/src/callbacks_step/save_solution_dg.jl b/src/callbacks_step/save_solution_dg.jl index 7367886ca94..0f9a8f80a5c 100644 --- a/src/callbacks_step/save_solution_dg.jl +++ b/src/callbacks_step/save_solution_dg.jl @@ -8,8 +8,7 @@ function save_solution_file(u, time, dt, timestep, mesh::Union{SerialTreeMesh, StructuredMesh, StructuredMeshView, - UnstructuredMesh2D, SerialP4estMesh, - SerialT8codeMesh}, + UnstructuredMesh2D, SerialP4estMesh}, equations, dg::DG, cache, solution_callback, element_variables = Dict{Symbol, Any}(), @@ -90,6 +89,76 @@ function save_solution_file(u, time, dt, timestep, return filename end +function save_solution_file(u, time, dt, timestep, + mesh::SerialT8codeMesh{3}, + equations, dg::DG, cache, + solution_callback, + element_variables = Dict{Symbol, Any}(), + node_variables = Dict{Symbol, Any}(); + system = "") + @unpack output_directory, solution_variables = solution_callback + + # Filename based on current time step + if isempty(system) + file_prefix = joinpath(output_directory, @sprintf("solution_%06d", timestep)) + else + file_prefix = joinpath(output_directory, + @sprintf("solution_%s_%06d", system, timestep)) + end + + # Convert to different set of variables if requested + if solution_variables === cons2cons + data = u + n_vars = nvariables(equations) + else + # Reinterpret the solution array as an array of conservative variables, + # compute the solution variables via broadcasting, and reinterpret the + # result as a plain array of floating point numbers + data = Array(reinterpret(eltype(u), + solution_variables.(reinterpret(SVector{nvariables(equations), + eltype(u)}, u), + Ref(equations)))) + + # Find out variable count by looking at output from `solution_variables` function + n_vars = size(data, 1) + end + + # Save the complete connectivity and `p4est` data to disk. + num_elements = ncells(mesh) + + # We need to allocate a new array to store the data on their own. + # These arrays have one entry per local element. + vtk_array = Vector{Cdouble}(undef, num_elements) + + # Copy the element's volumes from our data array to the output array. + for element in 1:num_elements + mean = zero(real(dg.basis)) + + for k in eachnode(dg), j in eachnode(dg), i in eachnode(dg) + u_local = get_node_vars(data, equations, dg, i, j, k, element) + mean += u_local[1] * dg.basis.weights[i]*dg.basis.weights[j]*dg.basis.weights[k] * 0.125 + end + + vtk_array[element] = mean + end + + # WARNING: This code hangs for Julia v1.8.* or older. Use at least Julia v1.9. + vtk_data = [ + t8_vtk_data_field_t(T8_VTK_SCALAR, + NTuple{8192, Cchar}(rpad("rho_alpha\0", 8192, ' ')), + pointer(vtk_array)), + ] + + # The number of user defined data fields to write. + num_data = length(vtk_data) + + Trixi.t8_forest_write_vtk_ext(mesh.forest, file_prefix, 0, 0, 0, 0, 0, 1, 0, num_data, pointer(vtk_data)) + + return file_prefix +end + + + function save_solution_file(u, time, dt, timestep, mesh::Union{ParallelTreeMesh, ParallelP4estMesh}, equations, dg::DG, cache, diff --git a/src/equations/equations.jl b/src/equations/equations.jl index f98bc01c9d5..8dc658679f6 100644 --- a/src/equations/equations.jl +++ b/src/equations/equations.jl @@ -511,6 +511,7 @@ abstract type AbstractEquationsParabolic{NDIMS, NVARS, GradientVariables} <: abstract type AbstractThreeEquationsEquations{NDIMS, NVARS} <: AbstractEquations{NDIMS, NVARS} end include("three_equations_2d.jl") +include("three_equations_3d.jl") # Lighthill-Witham-Richards (LWR) traffic flow model abstract type AbstractTrafficFlowLWREquations{NDIMS, NVARS} <: diff --git a/src/equations/three_equations_3d.jl b/src/equations/three_equations_3d.jl new file mode 100644 index 00000000000..41320659a19 --- /dev/null +++ b/src/equations/three_equations_3d.jl @@ -0,0 +1,411 @@ +# By default, Julia/LLVM does not use fused multiply-add operations (FMAs). +# Since these FMAs can increase the performance of many numerical algorithms, +# we need to opt-in explicitly. +# See https://ranocha.de/blog/Optimizing_EC_Trixi for further details. +@muladd begin + + +@doc raw""" + ThreeEquations2D(gamma) + + The ThreeEquations equations + """ +struct ThreeEquations3D{RealT<:Real} <: AbstractThreeEquationsEquations{3, 6} + gamma::RealT + k0::RealT + rho_0::RealT + gravity::RealT + + function ThreeEquations3D(gamma, k0, rho_0, gravity) + return new{typeof(gamma)}(gamma, k0, rho_0, gravity) + end +end + + +have_nonconservative_terms(::ThreeEquations3D) = True() +varnames(::typeof(cons2cons), ::ThreeEquations3D) = ("alpha_rho", "alpha_rho_v1", "alpha_rho_v2", "alpha_rho_v3", "alpha", "phi") +varnames(::typeof(cons2prim), ::ThreeEquations3D) = ("rho", "v1", "v2","v3", "alpha", "phi") + + +# Set initial conditions at physical location `x` for time `t` +""" + initial_condition_constant(x, t, equations::ThreeEquations3D) + +A constant initial condition to test free-stream preservation. +""" +function initial_condition_constant(x, t, equations::ThreeEquations3D) + alpha_rho = 1000.0 + alpha_rho_v1 = 0.0 + alpha_rho_v2 = 0.0 + alpha_rho_v3 = 0.0 + alpha = 1.0 + phi = x[2] + return SVector(alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha_rho_v3, alpha, phi) +end + +function source_terms_gravity(u, x, t, equations::ThreeEquations3D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha = u + du1 = 0.0 + du2 = 0.0 + du3 = -alpha_rho * equations.gravity + du4 = 0.0 + du5 = 0.0 + du6 = 0.0 + + return SVector(du1, du2, du3, du4, du5, du6) +end + + +function boundary_condition_wall(u_inner, orientation, + direction, x, t, + surface_flux_function, + equations::ThreeEquations3D) + + # Boundary state is equal to the inner state except for the velocity. For boundaries + # in the -x/+x direction, we multiply the velocity in the x direction by -1. + # Similarly, for boundaries in the -y/+y direction, we multiply the velocity in the + # y direction by -1 + if direction in (1, 2) # x direction + u_boundary = SVector(u_inner[1], -u_inner[2], u_inner[3], u_inner[4], u_inner[5], u_inner[6]) + elseif direction in (3,4) # y direction + u_boundary = SVector(u_inner[1], u_inner[2], -u_inner[3], u_inner[4], u_inner[5], u_inner[6]) + else + u_boundary = SVector(u_inner[1], u_inner[2], u_inner[3], -u_inner[4], u_inner[5], u_inner[6]) + end + + # Calculate boundary flux + if iseven(direction) # u_inner is "left" of boundary, u_boundary is "right" of boundary + flux = surface_flux_function(u_inner, u_boundary, orientation, equations) + else # u_boundary is "left" of boundary, u_inner is "right" of boundary + flux = surface_flux_function(u_boundary, u_inner, orientation, equations) + end + + return flux +end + +@inline function boundary_condition_wall(u_inner, normal_direction::AbstractVector, + x, t, + surface_flux_function, + equations::ThreeEquations3D) + + # normalize the outward pointing direction + normal = normal_direction / norm(normal_direction) + + # compute the normal velocity + u_normal = normal[1] * u_inner[2] + normal[2] * u_inner[3] + normal[3] * u_inner[4] + + # create the "external" boundary solution state + u_boundary = SVector(u_inner[1], + u_inner[2] - 2.0 * u_normal * normal[1], + u_inner[3] - 2.0 * u_normal * normal[2], + u_inner[4] - 2.0 * u_normal * normal[3], + u_inner[5], + u_inner[6]) + + # calculate the boundary flux + flux = surface_flux_function(u_inner, u_boundary, normal_direction, equations) + + return flux +end + + +# Calculate 3D flux for a single point +@inline function flux(u, orientation::Integer, equations::ThreeEquations3D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha_rho_v3, alpha, phi = u + v1 = alpha_rho_v1 / alpha_rho + v2 = alpha_rho_v2 / alpha_rho + v3 = alpha_rho_v3 / alpha_rho + p = pressure(u, equations) + if orientation == 1 + f1 = alpha_rho_v1 + f2 = alpha_rho_v1 * v1 + alpha * p + f3 = alpha_rho_v1 * v2 + f4 = alpha_rho_v1 * v3 + f5 = 0.0 + f6 = 0.0 + elseif orientation == 2 + f1 = alpha_rho_v2 + f2 = alpha_rho_v1 * v2 + f3 = alpha_rho_v2 * v2 + alpha * p + f4 = alpha_rho_v2 * v3 + f5 = 0.0 + f6 = 0.0 + else + f1 = alpha_rho_v3 + f2 = alpha_rho_v3 * v1 + f3 = alpha_rho_v3 * v2 + f4 = alpha_rho_v3 * v3 + alpha * p + f5 = 0.0 + f6 = 0.0 + end + return SVector(f1, f2, f3, f4, f5, f6) +end + +# Calculate 3D flux for a single point in the normal direction +# Note, this directional vector is not normalized +@inline function flux(u, normal_direction::AbstractVector, equations::ThreeEquations3D) + rho, v1, v2, v3, alpha, phi = cons2prim(u, equations) + + v_normal = v1 * normal_direction[1] + v2 * normal_direction[2] + v3 * normal_direction[3] + rho_v_normal = rho * v_normal + p = pressure(u, equations) + + f1 = alpha * rho_v_normal + f2 = alpha * rho_v_normal * v1 + alpha * p * normal_direction[1] + f3 = alpha * rho_v_normal * v2 + alpha * p * normal_direction[2] + f4 = alpha * rho_v_normal * v3 + alpha * p * normal_direction[3] + f5 = 0.0 + f6 = 0.0 + + return SVector(f1, f2, f3, f4, f5, f6) +end + +@inline function flux_nonconservative_ThreeEquations(u_ll, u_rr, orientation::Integer, equations::ThreeEquations3D) + + v1_ll = u_ll[2]/u_ll[1] + v2_ll = u_ll[3]/u_ll[1] + v3_ll = u_ll[4]/u_ll[1] + alpha_rr = u_rr[5] + + z = zero(eltype(u_ll)) + + if orientation == 1 + f = SVector(z, z, z, z, v1_ll * alpha_rr, z) + elseif orientation == 2 + f = SVector(z, z, z, z, v2_ll * alpha_rr, z) + else + f = SVector(z, z, z, z, v3_ll * alpha_rr, z) + end + + return f +end + + +@inline function flux_nonconservative_ThreeEquations(u_ll, u_rr, + normal_direction_ll::AbstractVector, + normal_direction_average::AbstractVector, + equations::ThreeEquations3D) + + v1_ll = u_ll[2]/u_ll[1] + v2_ll = u_ll[3]/u_ll[1] + v3_ll = u_ll[4]/u_ll[1] + alpha_rr = u_rr[5] + + v_dot_n_ll = v1_ll * normal_direction_ll[1] + v2_ll * normal_direction_ll[2] + v3_ll * normal_direction_ll[3] + + z = zero(eltype(u_ll)) + + f = SVector(z, z, z, z, v_dot_n_ll * alpha_rr, z) + + return f + +end + +@inline function flux_nonconservative_ThreeEquations_well(u_ll, u_rr, orientation::Integer, equations::ThreeEquations3D) + + v1_ll = u_ll[2]/u_ll[1] + v2_ll = u_ll[3]/u_ll[1] + v3_ll = u_ll[4]/u_ll[1] + alpha_rr = u_rr[5] + phi_ll = u_ll[6] + phi_rr = u_rr[6] + + gravity = u_ll[1] * equations.gravity * phi_rr + well_balanced = u_ll[1]/equations.rho_0 * equations.k0 * exp(equations.rho_0 * equations.gravity * phi_ll/equations.k0) * exp(-equations.rho_0 * equations.gravity * phi_rr/equations.k0) + + z = zero(eltype(u_ll)) + + if orientation == 1 + f = SVector(z, z, z, z, v1_ll * alpha_rr, z) + elseif orientation == 2 + f = SVector(z, z, -well_balanced, z, v2_ll * alpha_rr, z) + else + f = SVector(z, z, z, z, v3_ll * alpha_rr, z) + end + + return f +end + + +@inline function flux_nonconservative_ThreeEquations_well(u_ll, u_rr, + normal_direction_ll::AbstractVector, + normal_direction_average::AbstractVector, + equations::ThreeEquations3D) + + v1_ll = u_ll[2]/u_ll[1] + v2_ll = u_ll[3]/u_ll[1] + v3_ll = u_ll[4]/u_ll[1] + alpha_rr = u_rr[5] #* normal_direction_average[1] + u_rr[4] * normal_direction_average[2] + phi_ll = u_ll[6] + phi_rr = u_rr[6] + + v_dot_n_ll = v1_ll * normal_direction_ll[1] + v2_ll * normal_direction_ll[2] + v3_ll * normal_direction_ll[3] + + well_balanced = u_ll[1]/equations.rho_0 * equations.k0 * exp(equations.rho_0 * equations.gravity * phi_ll/equations.k0) * exp(-equations.rho_0 * equations.gravity * phi_rr/equations.k0) * normal_direction_average[2] + gravity = u_ll[1] * equations.gravity * u_rr[5] * normal_direction_average[2] + + z = zero(eltype(u_ll)) + + f = SVector(z, z, -well_balanced, z, v_dot_n_ll * alpha_rr, z) + + return f + +end + + +# Calculate maximum wave speed for local Lax-Friedrichs-type dissipation as the +# maximum velocity magnitude plus the maximum speed of sound +@inline function max_abs_speed_naive(u_ll, u_rr, orientation::Integer, equations::ThreeEquations3D) + rho_ll, v1_ll, v2_ll, v3_ll, alpha_ll, phi_ll = cons2prim(u_ll, equations) + rho_rr, v1_rr, v2_rr, v3_rr, alpha_rr, phi_rr = cons2prim(u_rr, equations) + + # Get the velocity value in the appropriate direction + if orientation == 1 + v_ll = v1_ll + v_rr = v1_rr + elseif orientation == 2 # orientation == 2 + v_ll = v2_ll + v_rr = v2_rr + else + v_ll = v3_ll + v_rr = v3_rr + end + # Calculate sound speeds + c_ll = sqrt(equations.gamma * (equations.k0 / equations.rho_0) * (rho_ll/equations.rho_0)^(equations.gamma - 1)) + c_rr = sqrt(equations.gamma * (equations.k0 / equations.rho_0) * (rho_rr/equations.rho_0)^(equations.gamma - 1)) + + λ_max = max(abs(v_ll), abs(v_rr)) + max(c_ll, c_rr) +end + +@inline function max_abs_speed_naive(u_ll, u_rr, normal_direction::AbstractVector, equations::ThreeEquations3D) + rho_ll, v1_ll, v2_ll, v3_ll, alpha_ll, phi_ll = cons2prim(u_ll, equations) + rho_rr, v1_rr, v2_rr, v3_rr, alpha_rr, phi_rr = cons2prim(u_rr, equations) + + # Calculate normal velocities and sound speed + # left + v_ll = ( v1_ll * normal_direction[1] + + v2_ll * normal_direction[2] + v3_ll * normal_direction[3] ) + c_ll = sqrt(equations.gamma * (equations.k0 / equations.rho_0) * (rho_ll/equations.rho_0)^(equations.gamma - 1)) + # right + v_rr = ( v1_rr * normal_direction[1] + + v2_rr * normal_direction[2] + + v3_rr * normal_direction[3]) + c_rr = sqrt(equations.gamma * (equations.k0 / equations.rho_0) * (rho_rr/equations.rho_0)^(equations.gamma - 1)) + + return max(abs(v_ll), abs(v_rr)) + max(c_ll, c_rr) * norm(normal_direction) +end + + + +# Calculate minimum and maximum wave speeds for HLL-type fluxes +@inline function min_max_speed_naive(u_ll, u_rr, orientation::Integer, equations::ThreeEquations3D) + rho_ll, v1_ll, v2_ll, v3_ll, alpha_ll, phi_ll = cons2prim(u_ll, equations) + rho_rr, v1_rr, v2_rr, v3_rr, alpha_rr, phi_rr = cons2prim(u_rr, equations) + + if orientation == 1 # x-direction + λ_min = v1_ll - sqrt(equations.gamma * (equations.k0 / equations.rho_0) * (rho_ll/equations.rho_0)^(equations.gamma - 1)) + λ_max = v1_rr + sqrt(equations.gamma * (equations.k0 / equations.rho_0) * (rho_rr/equations.rho_0)^(equations.gamma - 1)) + elseif orientation == 2 # y-direction + λ_min = v2_ll - sqrt(equations.gamma * (equations.k0 / equations.rho_0) * (rho_ll/equations.rho_0)^(equations.gamma - 1)) + λ_max = v2_rr + sqrt(equations.gamma * (equations.k0 / equations.rho_0) * (rho_rr/equations.rho_0)^(equations.gamma - 1)) + else + λ_min = v3_ll - sqrt(equations.gamma * (equations.k0 / equations.rho_0) * (rho_ll/equations.rho_0)^(equations.gamma - 1)) + λ_max = v3_rr + sqrt(equations.gamma * (equations.k0 / equations.rho_0) * (rho_rr/equations.rho_0)^(equations.gamma - 1)) + end + + return λ_min, λ_max +end + +@inline function min_max_speed_naive(u_ll, u_rr, normal_direction::AbstractVector, + equations::ThreeEquations3D) + rho_ll, v1_ll, v2_ll, v3_ll, alpha_ll, phi_ll = cons2prim(u_ll, equations) + rho_rr, v1_rr, v2_rr, v3_rr, alpha_rr, phi_rr = cons2prim(u_rr, equations) + + v_normal_ll = v1_ll * normal_direction[1] + v2_ll * normal_direction[2] + v3_ll * normal_direction[3] + v_normal_rr = v1_rr * normal_direction[1] + v2_rr * normal_direction[2] + v3_rr * normal_direction[3] + + norm_ = norm(normal_direction) + # The v_normals are already scaled by the norm + λ_min = v_normal_ll - sqrt(equations.gamma * (equations.k0 / equations.rho_0) * (rho_ll/equations.rho_0)^(equations.gamma - 1)) * norm_ + λ_max = v_normal_rr + sqrt(equations.gamma * (equations.k0 / equations.rho_0) * (rho_rr/equations.rho_0)^(equations.gamma - 1)) * norm_ + + return λ_min, λ_max +end + + +@inline function max_abs_speeds(u, equations::ThreeEquations3D) + rho, v1, v2, v3, alpha, phi = cons2prim(u, equations) + c = sqrt(equations.gamma * (equations.k0 / equations.rho_0) * (rho/equations.rho_0)^(equations.gamma - 1)) + + return abs(v1) + c, abs(v2) + c, abs(v3) + c +end + + +# Convert conservative variables to primitive +@inline function cons2prim(u, equations::ThreeEquations3D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha_rho_v3, alpha, phi = u + + rho = alpha_rho/alpha + v1 = alpha_rho_v1 / alpha_rho + v2 = alpha_rho_v2 / alpha_rho + v3 = alpha_rho_v3 / alpha_rho + + return SVector(rho, v1, v2, v3, alpha, phi) +end + +# Convert conservative variables to primitive +@inline function cons2entropy(u, equations::ThreeEquations3D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha_rho_v3, alpha, phi = u + + rho = alpha_rho/alpha + v1 = alpha_rho_v1 / alpha_rho + v2 = alpha_rho_v2 / alpha_rho + v3 = alpha_rho_v3 / alpha_rho + + return SVector(rho, v1, v2, v3, alpha, phi) +end + +# Convert primitive to conservative variables +@inline function prim2cons(prim, equations::ThreeEquations3D) + rho, v1, v2, v3, alpha, phi = prim + alpha_rho = rho * alpha + alpha_rho_v1 = alpha_rho * v1 + alpha_rho_v2 = alpha_rho * v2 + alpha_rho_v3 = alpha_rho * v3 + return SVector(alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha_rho_v3, alpha, phi) +end + +@inline function density(u, equations::ThreeEquations3D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha_rho_v3, alpha, phi = u + return alpha_rho/alpha +end + +@inline function alpha_rho(u, equations::ThreeEquations3D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha_rho_v3, alpha, phi = u + return alpha_rho +end + +@inline function pressure(u, equations::ThreeEquations3D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha_rho_v3, alpha, phi = u + if alpha < 0.1 + p = 0.0 + else + p = equations.k0 * ((alpha_rho/equations.rho_0)^(equations.gamma) - 1) + end + return p +end + +@inline function density_pressure(u, equations::ThreeEquations3D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha_rho_v3, alpha, phi = u + rho = alpha_rho / alpha + rho_times_p = pressure(u,equations) * rho + return rho_times_p +end + +# Calculate the error for the "water-at-rest" test case +@inline function water_at_rest_error(u, equations::ThreeEquations3D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha_rho_v3, alpha, phi = u + rho0 = equations.rho_0 * exp(-(equations.gravity * equations.rho_0/equations.k0) * (phi - 1.0)) + return abs(alpha_rho/alpha - rho0) +end + +end # @muladd diff --git a/src/meshes/mesh_io.jl b/src/meshes/mesh_io.jl index d74a0c0cea1..ceec59500b5 100644 --- a/src/meshes/mesh_io.jl +++ b/src/meshes/mesh_io.jl @@ -227,9 +227,23 @@ end # TODO: Implement this function as soon as there is support for this in `t8code`. function save_mesh_file(mesh::T8codeMesh, output_directory, timestep, mpi_parallel) - error("Mesh file output not supported yet for `T8codeMesh`.") + # error("Mesh file output not supported yet for `T8codeMesh`.") - return joinpath(output_directory, "dummy_mesh.h5") + # Create output directory (if it does not exist) + mkpath(output_directory) + + # Determine file name based on existence of meaningful time step + if timestep > 0 + prefix = joinpath(output_directory, @sprintf("mesh_%06d", timestep)) + else + prefix = joinpath(output_directory, "mesh") + end + + file_prefix = joinpath(output_directory, prefix) + + Trixi.t8_forest_write_vtk_ext(mesh.forest, file_prefix, 0, 0, 0, 0, 0, 1, 0, 0, C_NULL) + + return file_prefix end """ diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 0af4c6ae023..f4ca2fe17ce 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -26,6 +26,8 @@ mutable struct T8codeMesh{NDIMS, RealT <: Real, IsParallel, NDIMSP2, NNODES} <: nmpiinterfaces :: Int nmpimortars :: Int + unsaved_changes::Bool + function T8codeMesh{NDIMS}(forest::Ptr{t8_forest}, tree_node_coordinates, nodes, boundary_names, current_filename) where {NDIMS} @@ -38,6 +40,7 @@ mutable struct T8codeMesh{NDIMS, RealT <: Real, IsParallel, NDIMSP2, NNODES} <: mesh.boundary_names = boundary_names mesh.current_filename = current_filename mesh.tree_node_coordinates = tree_node_coordinates + mesh.unsaved_changes = true finalizer(mesh) do mesh # When finalizing `mesh.forest`, `mesh.scheme` and `mesh.cmesh` are diff --git a/src/solvers/dgsem_p4est/dg_3d.jl b/src/solvers/dgsem_p4est/dg_3d.jl index 5b3c5ae5ca8..856ffc84fcf 100644 --- a/src/solvers/dgsem_p4est/dg_3d.jl +++ b/src/solvers/dgsem_p4est/dg_3d.jl @@ -359,7 +359,9 @@ function calc_boundary_flux!(cache, t, boundary_condition, boundary_indexing, equations, surface_integral, dg::DG) @unpack boundaries = cache @unpack surface_flux_values, node_coordinates, contravariant_vectors = cache.elements - @unpack surface_flux = surface_integral + # @unpack surface_flux = surface_integral + surface_flux, nonconservative_flux = surface_integral.surface_flux + index_range = eachnode(dg) @threaded for local_index in eachindex(boundary_indexing) @@ -400,9 +402,12 @@ function calc_boundary_flux!(cache, t, boundary_condition, boundary_indexing, flux_ = boundary_condition(u_inner, normal_direction, x, t, surface_flux, equations) + noncons_ = nonconservative_flux(u_inner, u_inner, normal_direction, + normal_direction, equations) + # Copy flux to element storage in the correct orientation for v in eachvariable(equations) - surface_flux_values[v, i, j, direction, element] = flux_[v] + surface_flux_values[v, i, j, direction, element] = flux_[v] + 0.5*noncons_[v] end i_node += i_node_step_i diff --git a/src/solvers/dgsem_tree/dg_2d.jl b/src/solvers/dgsem_tree/dg_2d.jl index a9bc217aa1e..15729cb8046 100644 --- a/src/solvers/dgsem_tree/dg_2d.jl +++ b/src/solvers/dgsem_tree/dg_2d.jl @@ -1151,58 +1151,58 @@ function calc_sources!(du, u, t, source_terms::Nothing, return nothing end -# function calc_sources!(du, u, t, source_terms, -# equations::AbstractEquations{2}, dg::DG, cache) -# @unpack node_coordinates = cache.elements -# -# @threaded for element in eachelement(dg, cache) -# for j in eachnode(dg), i in eachnode(dg) -# u_local = get_node_vars(u, equations, dg, i, j, element) -# x_local = get_node_coords(node_coordinates, equations, dg, -# i, j, element) -# du_local = source_terms(u_local, x_local, t, equations) -# add_to_node_vars!(du, du_local, equations, dg, i, j, element) -# end -# end -# -# return nothing -# end - function calc_sources!(du, u, t, source_terms, equations::AbstractEquations{2}, dg::DG, cache) @unpack node_coordinates = cache.elements @threaded for element in eachelement(dg, cache) - source_terms(du, u, node_coordinates, t, element, equations, dg) - - # N = length(dg.basis.nodes) - # u_local = Array{eltype(u),3}(undef, nvariables(equations), N, N) - # x_local = Array{eltype(u),3}(undef, 2, N, N) - # z = zero(eltype(u)) - - # for j in eachnode(dg), i in eachnode(dg) - # u_local[:,i,j] = get_node_vars(u, equations, dg, i, j, element) - # x_local[:,i,j] = get_node_coords(node_coordinates, equations, dg, - # i, j, element) - # end - - # # du_local = source_terms(u_local, x_local, t, equations, dg) - # # display(du_local) - - # for j in eachnode(dg), i in eachnode(dg) - - # # du[3,:,:] .= -u[1,:,:] * 9.81 - # # sv = SVector(du_local[:,i,j]...) - - # sv = SVector(z, z, -u_local[1,i,j] * 9.81, z, z) - # # println(sv) - # # add_to_node_vars!(du, @view(du_local[:,i,j]), equations, dg, i, j, element) - # # add_to_node_vars!(du, du_local[:,i,j], equations, dg, i, j, element) - # add_to_node_vars!(du, sv, equations, dg, i, j, element) - # end + for j in eachnode(dg), i in eachnode(dg) + u_local = get_node_vars(u, equations, dg, i, j, element) + x_local = get_node_coords(node_coordinates, equations, dg, + i, j, element) + du_local = source_terms(u_local, x_local, t, equations) + add_to_node_vars!(du, du_local, equations, dg, i, j, element) + end end return nothing end +## function calc_sources!(du, u, t, source_terms, +## equations::AbstractEquations{2}, dg::DG, cache) +## @unpack node_coordinates = cache.elements +## +## @threaded for element in eachelement(dg, cache) +## source_terms(du, u, node_coordinates, t, element, equations, dg) +## +## # N = length(dg.basis.nodes) +## # u_local = Array{eltype(u),3}(undef, nvariables(equations), N, N) +## # x_local = Array{eltype(u),3}(undef, 2, N, N) +## # z = zero(eltype(u)) +## +## # for j in eachnode(dg), i in eachnode(dg) +## # u_local[:,i,j] = get_node_vars(u, equations, dg, i, j, element) +## # x_local[:,i,j] = get_node_coords(node_coordinates, equations, dg, +## # i, j, element) +## # end +## +## # # du_local = source_terms(u_local, x_local, t, equations, dg) +## # # display(du_local) +## +## # for j in eachnode(dg), i in eachnode(dg) +## +## # # du[3,:,:] .= -u[1,:,:] * 9.81 +## # # sv = SVector(du_local[:,i,j]...) +## +## # sv = SVector(z, z, -u_local[1,i,j] * 9.81, z, z) +## # # println(sv) +## # # add_to_node_vars!(du, @view(du_local[:,i,j]), equations, dg, i, j, element) +## # # add_to_node_vars!(du, du_local[:,i,j], equations, dg, i, j, element) +## # add_to_node_vars!(du, sv, equations, dg, i, j, element) +## # end +## end +## +## return nothing +## end + end # @muladd diff --git a/src/solvers/dgsem_tree/indicators_3d.jl b/src/solvers/dgsem_tree/indicators_3d.jl index a11a8e06e4b..e271f4774f8 100644 --- a/src/solvers/dgsem_tree/indicators_3d.jl +++ b/src/solvers/dgsem_tree/indicators_3d.jl @@ -251,4 +251,41 @@ function (indicator_max::IndicatorMax)(u::AbstractArray{<:Any, 5}, return alpha end + +function create_cache(::Type{IndicatorClamp}, equations::AbstractEquations{3}, + basis::LobattoLegendreBasis) + alpha = Vector{real(basis)}() + return (; alpha, basis.weights) +end + +function create_cache(typ::Type{IndicatorClamp}, mesh, equations::AbstractEquations{3}, + dg::DGSEM, cache) + cache = create_cache(typ, equations, dg.basis) +end + +function (indicator_clamp::IndicatorClamp)(u::AbstractArray{<:Any, 5}, + mesh, equations, dg::DGSEM, cache; + kwargs...) + @unpack alpha, weights = indicator_clamp.cache + resize!(alpha, nelements(dg, cache)) + + @threaded for element in eachelement(dg, cache) + mean = zero(real(dg.basis)) + + for k in eachnode(dg), j in eachnode(dg), i in eachnode(dg) + u_local = get_node_vars(u, equations, dg, i, j, k, element) + mean += indicator_clamp.variable(u_local, equations) * weights[i] * + weights[j] * weights[k] + end + mean *= 0.125 + + if indicator_clamp.min <= mean <= indicator_clamp.max + alpha[element] = 1.0 + else + alpha[element] = -1.0 + end + end + + return alpha +end end # @muladd From 08fd6567618006ab1a595f35592751fb7c7150ff Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Wed, 7 Aug 2024 09:12:31 +0200 Subject: [PATCH 80/81] Backup. --- ...ixir_advection_amr_solution_independent.jl | 4 + .../elixir_advection_amr_unstructured_flag.jl | 4 + .../t8code_2d_dgsem/elixir_advection_basic.jl | 4 + .../elixir_advection_extended.jl | 4 + .../elixir_advection_nonconforming_flag.jl | 4 + .../elixir_advection_restart.jl | 4 + .../elixir_advection_restart_amr.jl | 4 + .../elixir_advection_unstructured_flag.jl | 4 + .../elixir_euler_free_stream.jl | 4 + .../t8code_2d_dgsem/elixir_euler_sedov.jl | 4 + .../elixir_euler_shockcapturing_ec.jl | 4 + ...e_terms_nonconforming_unstructured_flag.jl | 4 + .../elixir_eulergravity_convergence.jl | 4 + .../t8code_2d_dgsem/elixir_mhd_alfven_wave.jl | 4 + examples/t8code_2d_dgsem/elixir_mhd_rotor.jl | 4 + .../elixir_shallowwater_source_terms.jl | 4 + .../t8code_3d_dgsem/elixir_advection_amr.jl | 4 + ...lixir_advection_amr_unstructured_curved.jl | 8 +- .../t8code_3d_dgsem/elixir_advection_basic.jl | 4 + .../elixir_advection_nonconforming.jl | 4 + .../elixir_advection_restart.jl | 4 + .../elixir_advection_unstructured_curved.jl | 4 + examples/t8code_3d_dgsem/elixir_euler_ec.jl | 4 + .../elixir_euler_free_stream.jl | 4 + .../elixir_euler_free_stream_extruded.jl | 4 + .../t8code_3d_dgsem/elixir_euler_sedov.jl | 4 + ...terms_nonconforming_unstructured_curved.jl | 4 + .../elixir_euler_source_terms_nonperiodic.jl | 4 + src/auxiliary/t8code.jl | 19 ++-- src/callbacks_step/save_solution_dg.jl | 2 +- src/meshes/t8code_mesh.jl | 49 ++++----- src/solvers/dgsem_p4est/dg_2d_parallel.jl | 55 +++++++++ src/solvers/dgsem_p4est/dg_3d.jl | 104 ++++++++++++++---- src/solvers/dgsem_p4est/dg_3d_parallel.jl | 61 ++++++++++ 34 files changed, 340 insertions(+), 66 deletions(-) diff --git a/examples/t8code_2d_dgsem/elixir_advection_amr_solution_independent.jl b/examples/t8code_2d_dgsem/elixir_advection_amr_solution_independent.jl index cfa0a34bb34..f7e5cc39506 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_amr_solution_independent.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_amr_solution_independent.jl @@ -152,3 +152,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback save_everystep = false, callback = callbacks); summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl b/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl index bd82809e016..b6d92d99c2a 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_amr_unstructured_flag.jl @@ -93,3 +93,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), save_everystep = false, callback = callbacks); summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_2d_dgsem/elixir_advection_basic.jl b/examples/t8code_2d_dgsem/elixir_advection_basic.jl index 26ced0970fe..2c19ace1dea 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_basic.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_basic.jl @@ -55,3 +55,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), # Print the timer summary summary_callback() + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_2d_dgsem/elixir_advection_extended.jl b/examples/t8code_2d_dgsem/elixir_advection_extended.jl index f7e06dd517e..034197ce9d8 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_extended.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_extended.jl @@ -83,3 +83,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), # Print the timer summary summary_callback() + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl b/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl index 7230c8c0b9e..cb14a7c23ae 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_nonconforming_flag.jl @@ -90,3 +90,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), # Print the timer summary summary_callback() + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_2d_dgsem/elixir_advection_restart.jl b/examples/t8code_2d_dgsem/elixir_advection_restart.jl index 6fb14c60038..9438eb4c38f 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_restart.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_restart.jl @@ -41,3 +41,7 @@ load_timestep!(integrator, restart_filename) sol = solve!(integrator) summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl b/examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl index 2c0f14c8d10..50b04a78fd7 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_restart_amr.jl @@ -60,3 +60,7 @@ load_timestep!(integrator, restart_filename) sol = solve!(integrator) summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl b/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl index aed6c755d5c..f13f0fae05a 100644 --- a/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_advection_unstructured_flag.jl @@ -76,3 +76,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), # Print the timer summary. summary_callback() + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_2d_dgsem/elixir_euler_free_stream.jl b/examples/t8code_2d_dgsem/elixir_euler_free_stream.jl index b2d49e3ccfe..349678f1637 100644 --- a/examples/t8code_2d_dgsem/elixir_euler_free_stream.jl +++ b/examples/t8code_2d_dgsem/elixir_euler_free_stream.jl @@ -91,3 +91,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback save_everystep = false, callback = callbacks); summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_2d_dgsem/elixir_euler_sedov.jl b/examples/t8code_2d_dgsem/elixir_euler_sedov.jl index fae7d818ad8..e8ec3a7a9e6 100644 --- a/examples/t8code_2d_dgsem/elixir_euler_sedov.jl +++ b/examples/t8code_2d_dgsem/elixir_euler_sedov.jl @@ -98,3 +98,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback save_everystep = false, callback = callbacks); summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_2d_dgsem/elixir_euler_shockcapturing_ec.jl b/examples/t8code_2d_dgsem/elixir_euler_shockcapturing_ec.jl index 2a0c4a6ee20..a7d10006593 100644 --- a/examples/t8code_2d_dgsem/elixir_euler_shockcapturing_ec.jl +++ b/examples/t8code_2d_dgsem/elixir_euler_shockcapturing_ec.jl @@ -70,3 +70,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback save_everystep = false, callback = callbacks); summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_2d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_flag.jl b/examples/t8code_2d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_flag.jl index 19502f1ce0e..9f9ad16dc9b 100644 --- a/examples/t8code_2d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_flag.jl +++ b/examples/t8code_2d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_flag.jl @@ -94,3 +94,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback save_everystep = false, callback = callbacks); summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_2d_dgsem/elixir_eulergravity_convergence.jl b/examples/t8code_2d_dgsem/elixir_eulergravity_convergence.jl index e63c1297882..1476dfa05c7 100644 --- a/examples/t8code_2d_dgsem/elixir_eulergravity_convergence.jl +++ b/examples/t8code_2d_dgsem/elixir_eulergravity_convergence.jl @@ -81,3 +81,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), save_everystep = false, callback = callbacks); summary_callback() # print the timer summary println("Number of gravity subcycles: ", semi.gravity_counter.ncalls_since_readout) + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_2d_dgsem/elixir_mhd_alfven_wave.jl b/examples/t8code_2d_dgsem/elixir_mhd_alfven_wave.jl index e184cb3fd05..04c36dd8642 100644 --- a/examples/t8code_2d_dgsem/elixir_mhd_alfven_wave.jl +++ b/examples/t8code_2d_dgsem/elixir_mhd_alfven_wave.jl @@ -57,3 +57,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback save_everystep = false, callback = callbacks); summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl b/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl index 88a2888de62..d171d13578e 100644 --- a/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl +++ b/examples/t8code_2d_dgsem/elixir_mhd_rotor.jl @@ -137,3 +137,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback save_everystep = false, callback = callbacks); summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_2d_dgsem/elixir_shallowwater_source_terms.jl b/examples/t8code_2d_dgsem/elixir_shallowwater_source_terms.jl index 688ddb2dbb5..0a937664493 100644 --- a/examples/t8code_2d_dgsem/elixir_shallowwater_source_terms.jl +++ b/examples/t8code_2d_dgsem/elixir_shallowwater_source_terms.jl @@ -59,3 +59,7 @@ callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, sav sol = solve(ode, RDPK3SpFSAL49(); abstol = 1.0e-8, reltol = 1.0e-8, ode_default_options()..., callback = callbacks); summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_3d_dgsem/elixir_advection_amr.jl b/examples/t8code_3d_dgsem/elixir_advection_amr.jl index d2a0e3fb666..6b9e41a17b4 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_amr.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_amr.jl @@ -77,3 +77,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback save_everystep = false, callback = callbacks); summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl b/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl index 56b2574849f..6964b5d85e3 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_amr_unstructured_curved.jl @@ -81,8 +81,8 @@ save_solution = SaveSolutionCallback(interval = 100, amr_controller = ControllerThreeLevel(semi, IndicatorMax(semi, variable = first), base_level = 1, - med_level = 2, med_threshold = 0.1, - max_level = 3, max_threshold = 0.6) + med_level = 3, med_threshold = 0.1, + max_level = 4, max_threshold = 0.6) amr_callback = AMRCallback(semi, amr_controller, interval = 5, adapt_initial_condition = true, @@ -112,3 +112,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback save_everystep = false, callback = callbacks); summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_3d_dgsem/elixir_advection_basic.jl b/examples/t8code_3d_dgsem/elixir_advection_basic.jl index ae97a73d182..415a9e78ba2 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_basic.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_basic.jl @@ -65,3 +65,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), # Print the timer summary summary_callback() + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_3d_dgsem/elixir_advection_nonconforming.jl b/examples/t8code_3d_dgsem/elixir_advection_nonconforming.jl index 0755a76ef45..f12c760bb1b 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_nonconforming.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_nonconforming.jl @@ -87,3 +87,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), # Print the timer summary summary_callback() + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_3d_dgsem/elixir_advection_restart.jl b/examples/t8code_3d_dgsem/elixir_advection_restart.jl index dfce8d2558a..0c158d72741 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_restart.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_restart.jl @@ -38,3 +38,7 @@ load_timestep!(integrator, restart_filename) sol = solve!(integrator) summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_3d_dgsem/elixir_advection_unstructured_curved.jl b/examples/t8code_3d_dgsem/elixir_advection_unstructured_curved.jl index f49137e054d..7c332e5b77a 100644 --- a/examples/t8code_3d_dgsem/elixir_advection_unstructured_curved.jl +++ b/examples/t8code_3d_dgsem/elixir_advection_unstructured_curved.jl @@ -96,3 +96,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), # Print the timer summary summary_callback() + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_3d_dgsem/elixir_euler_ec.jl b/examples/t8code_3d_dgsem/elixir_euler_ec.jl index e75b0f69636..2e13f7edd57 100644 --- a/examples/t8code_3d_dgsem/elixir_euler_ec.jl +++ b/examples/t8code_3d_dgsem/elixir_euler_ec.jl @@ -87,3 +87,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback save_everystep = false, callback = callbacks); summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_3d_dgsem/elixir_euler_free_stream.jl b/examples/t8code_3d_dgsem/elixir_euler_free_stream.jl index d45de658cc0..daf431a86ef 100644 --- a/examples/t8code_3d_dgsem/elixir_euler_free_stream.jl +++ b/examples/t8code_3d_dgsem/elixir_euler_free_stream.jl @@ -114,3 +114,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback save_everystep = false, callback = callbacks); summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_3d_dgsem/elixir_euler_free_stream_extruded.jl b/examples/t8code_3d_dgsem/elixir_euler_free_stream_extruded.jl index d24512a4cdd..19cdea7ee3c 100644 --- a/examples/t8code_3d_dgsem/elixir_euler_free_stream_extruded.jl +++ b/examples/t8code_3d_dgsem/elixir_euler_free_stream_extruded.jl @@ -102,3 +102,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), #maxiters=1 dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback save_everystep = false, callback = callbacks); summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_3d_dgsem/elixir_euler_sedov.jl b/examples/t8code_3d_dgsem/elixir_euler_sedov.jl index f897249ed2e..9f42ad24ac7 100644 --- a/examples/t8code_3d_dgsem/elixir_euler_sedov.jl +++ b/examples/t8code_3d_dgsem/elixir_euler_sedov.jl @@ -100,3 +100,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback save_everystep = false, callback = callbacks); summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_curved.jl b/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_curved.jl index 4b87b646df9..1346f603608 100644 --- a/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_curved.jl +++ b/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonconforming_unstructured_curved.jl @@ -116,3 +116,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback save_everystep = false, callback = callbacks); summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonperiodic.jl b/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonperiodic.jl index ce1662c8e50..612b1640239 100644 --- a/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonperiodic.jl +++ b/examples/t8code_3d_dgsem/elixir_euler_source_terms_nonperiodic.jl @@ -66,3 +66,7 @@ sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback save_everystep = false, callback = callbacks); summary_callback() # print the timer summary + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/src/auxiliary/t8code.jl b/src/auxiliary/t8code.jl index 86af91b5b51..fb0ddf287f8 100644 --- a/src/auxiliary/t8code.jl +++ b/src/auxiliary/t8code.jl @@ -18,6 +18,7 @@ function init_t8code() # `SC_LP_ERROR` to prevent a lot of output in AMR simulations # For development, log level `SC_LP_DEBUG` is recommended. LOG_LEVEL = T8code.Libt8.SC_LP_ERROR + # LOG_LEVEL = T8code.Libt8.SC_LP_DEBUG if T8code.Libt8.sc_is_initialized() == 0 # Initialize the sc library, has to happen before we initialize t8code. @@ -31,20 +32,16 @@ function init_t8code() if T8code.Libt8.p4est_is_initialized() == 0 T8code.Libt8.p4est_init(C_NULL, LOG_LEVEL) end + + MPI.add_finalize_hook!() do + status = T8code.Libt8.sc_finalize_noabort() + if status != 0 + @warn("Inconsistent state detected after finalizing t8code. Have you finalized all `T8codeMesh` objects and/or properly freed/un-referenced all t8code related objects?") + end + end # Initialize t8code. t8_init(LOG_LEVEL) - - if haskey(ENV, "TRIXI_T8CODE_SC_FINALIZE") - # Normally, `sc_finalize` should always be called during shutdown of an - # application. It checks whether there is still un-freed memory by t8code - # and/or T8code.jl and throws an exception if this is the case. For - # production runs this is not mandatory, but is helpful during - # development. Hence, this option is only activated when environment - # variable TRIXI_T8CODE_SC_FINALIZE exists. - @info "T8code.jl: `sc_finalize` will be called during shutdown of Trixi.jl." - MPI.add_finalize_hook!(T8code.Libt8.sc_finalize) - end else @warn "Preferences for T8code.jl are not set correctly. Until fixed, using `T8codeMesh` will result in a crash. " * "See also https://trixi-framework.github.io/Trixi.jl/stable/parallelization/#parallel_system_MPI" diff --git a/src/callbacks_step/save_solution_dg.jl b/src/callbacks_step/save_solution_dg.jl index 9d3cdc53677..96dc2231ac8 100644 --- a/src/callbacks_step/save_solution_dg.jl +++ b/src/callbacks_step/save_solution_dg.jl @@ -8,7 +8,7 @@ function save_solution_file(u, time, dt, timestep, mesh::Union{SerialTreeMesh, StructuredMesh, StructuredMeshView, - UnstructuredMesh2D, SerialP4estMesh}, + UnstructuredMesh2D, SerialP4estMesh, SerialT8codeMesh}, equations, dg::DG, cache, solution_callback, element_variables = Dict{Symbol, Any}(), diff --git a/src/meshes/t8code_mesh.jl b/src/meshes/t8code_mesh.jl index 5089a33bba8..904dbbdb08a 100644 --- a/src/meshes/t8code_mesh.jl +++ b/src/meshes/t8code_mesh.jl @@ -43,31 +43,14 @@ mutable struct T8codeMesh{NDIMS, RealT <: Real, IsParallel, NDIMSP2, NNODES} <: mesh.unsaved_changes = true finalizer(mesh) do mesh - # When finalizing `mesh.forest`, `mesh.scheme` and `mesh.cmesh` are + # When finalizing, `forest`, `scheme`, `cmesh`, and `geometry` are # also cleaned up from within `t8code`. The cleanup code for # `cmesh` does some MPI calls for deallocating shared memory # arrays. Due to garbage collection in Julia the order of shutdown - # is not deterministic. The following code might happen after MPI - # is already in finalized state. - # If the environment variable `TRIXI_T8CODE_SC_FINALIZE` is set the - # `finalize_hook` of the MPI module takes care of the cleanup. See - # further down. However, this might cause a pile-up of `mesh` - # objects during long-running sessions. - if !MPI.Finalized() - t8_forest_unref(Ref(mesh.forest)) - mesh.forest = C_NULL - end - end - - # This finalizer call is only recommended during development and not for - # production runs, especially long-running sessions since a reference to - # the `mesh` object will be kept throughout the lifetime of the session. - # See comments in `init_t8code()` in file `src/auxiliary/t8code.jl` for - # more information. - if haskey(ENV, "TRIXI_T8CODE_SC_FINALIZE") - MPI.add_finalize_hook!() do - t8_forest_unref(Ref(mesh.forest)) - end + # is not deterministic. Hence, "manual" finalization might be + # necessary in order to avoid MPI-related error output when closing + # the Julia program/session. + t8_forest_unref(Ref(mesh.forest)) end return mesh @@ -230,12 +213,14 @@ function T8codeMesh(ndims, ntrees, nelements, tree_node_coordinates, nodes, # Check if we are already in the next tree in terms of the `global_element_id`. global_tree_id = t8_forest_global_tree_id(forest, local_tree_id) + if global_element_id + 1 > cum_sum_num_elements_per_tree[global_tree_id + 1] return 0 end # Test if we already reached the targeted level. level = t8_element_level(eclass_scheme, elements[1]) + if level < levels[global_element_id + 1] return 1 # Go one refinement level deeper. end @@ -818,12 +803,15 @@ function adapt_callback_wrapper(forest, is_family, num_elements, elements_ptr)::Cint + passthrough = unsafe_pointer_to_objref(t8_forest_get_user_data(forest))[] elements = unsafe_wrap(Array, elements_ptr, num_elements) - return passthrough.adapt_callback(forest_from, which_tree, ts, lelement_id, elements, + result = passthrough.adapt_callback(forest_from, which_tree, ts, lelement_id, elements, Bool(is_family), passthrough.user_data) + + return result end """ @@ -874,12 +862,15 @@ function adapt(forest::Ptr{t8_forest}, adapt_callback; recursive = true, balance # Check out `examples/t8_step4_partition_balance_ghost.jl` in # https://github.com/DLR-AMR/T8code.jl for detailed explanations. let set_from = C_NULL, set_for_coarsening = 0, no_repartition = !partition - t8_forest_set_user_data(new_forest, - pointer_from_objref(Ref(adapt_callback_passthrough(adapt_callback, - user_data)))) + user_data_wrapper = adapt_callback_passthrough(adapt_callback, user_data) + user_data_pointer = pointer_from_objref(Ref(user_data_wrapper)) + + t8_forest_set_user_data(new_forest, user_data_pointer) + t8_forest_set_adapt(new_forest, forest, @t8_adapt_callback(adapt_callback_wrapper), recursive) + if balance t8_forest_set_balance(new_forest, set_from, no_repartition) end @@ -892,15 +883,17 @@ function adapt(forest::Ptr{t8_forest}, adapt_callback; recursive = true, balance t8_forest_set_ghost(new_forest, ghost, T8_GHOST_FACES) end - # The old forest is destroyed here. - # Call `t8_forest_ref(Ref(mesh.forest))` to keep it. + # Julias's GC leads to segfaults here. Temporarily switch it off. + GC.enable(false) t8_forest_commit(new_forest) + GC.enable(true) end return new_forest end function adapt!(mesh::T8codeMesh, adapt_callback; kwargs...) + # The old `mesh.forest` is destroyed here. # Call `t8_forest_ref(Ref(mesh.forest))` to keep it. mesh.forest = adapt(mesh.forest, adapt_callback; kwargs...) end diff --git a/src/solvers/dgsem_p4est/dg_2d_parallel.jl b/src/solvers/dgsem_p4est/dg_2d_parallel.jl index 3bf0cd0cab5..108502bdb96 100644 --- a/src/solvers/dgsem_p4est/dg_2d_parallel.jl +++ b/src/solvers/dgsem_p4est/dg_2d_parallel.jl @@ -133,6 +133,37 @@ end end end +# Inlined version of the interface flux computation for conservation laws +@inline function calc_mpi_interface_flux!(surface_flux_values, + mesh::Union{ParallelP4estMesh{2}, + ParallelT8codeMesh{2}}, + nonconservative_terms::True, equations, + surface_integral, dg::DG, cache, + interface_index, normal_direction, + interface_node_index, local_side, + surface_node_index, local_direction_index, + local_element_index) + @unpack u = cache.mpi_interfaces + surface_flux, nonconservative_flux = surface_integral.surface_flux + + u_ll, u_rr = get_surface_node_vars(u, equations, dg, interface_node_index, + interface_index) + + if local_side == 1 + flux_ = surface_flux(u_ll, u_rr, normal_direction, equations) + noncons_ = nonconservative_flux(u_ll, u_rr, normal_direction, + normal_direction, equations) + else # local_side == 2 + flux_ = -surface_flux(u_ll, u_rr, -normal_direction, equations) + noncons_ = -nonconservative_flux(u_rr, u_ll, -normal_direction, + -normal_direction, equations) + end + + for v in eachvariable(equations) + surface_flux_values[v, surface_node_index, local_direction_index, local_element_index] = flux_[v] + 0.5 * noncons_[v] + end +end + function prolong2mpimortars!(cache, u, mesh::Union{ParallelP4estMesh{2}, ParallelT8codeMesh{2}}, equations, @@ -275,6 +306,30 @@ end set_node_vars!(fstar[position_index], flux, equations, dg, node_index) end +# Inlined version of the mortar flux computation on small elements for conservation laws +@inline function calc_mpi_mortar_flux!(fstar, + mesh::Union{ParallelP4estMesh{2}, + ParallelT8codeMesh{2}}, + nonconservative_terms::True, equations, + surface_integral, dg::DG, cache, + mortar_index, position_index, normal_direction, + node_index) + @unpack u = cache.mpi_mortars + surface_flux, nonconservative_flux = surface_integral.surface_flux + + u_ll, u_rr = get_surface_node_vars(u, equations, dg, position_index, node_index, + mortar_index) + + flux_ = surface_flux(u_ll, u_rr, normal_direction, equations) + noncons_ = nonconservative_flux(u_ll, u_rr, normal_direction, + normal_direction, equations) + + flux = flux_ + 0.5 * noncons_ + + # Copy flux to buffer + set_node_vars!(fstar[position_index], flux, equations, dg, node_index) +end + @inline function mpi_mortar_fluxes_to_elements!(surface_flux_values, mesh::Union{ParallelP4estMesh{2}, ParallelT8codeMesh{2}}, diff --git a/src/solvers/dgsem_p4est/dg_3d.jl b/src/solvers/dgsem_p4est/dg_3d.jl index 856ffc84fcf..3f6cbbc2b3e 100644 --- a/src/solvers/dgsem_p4est/dg_3d.jl +++ b/src/solvers/dgsem_p4est/dg_3d.jl @@ -359,8 +359,7 @@ function calc_boundary_flux!(cache, t, boundary_condition, boundary_indexing, equations, surface_integral, dg::DG) @unpack boundaries = cache @unpack surface_flux_values, node_coordinates, contravariant_vectors = cache.elements - # @unpack surface_flux = surface_integral - surface_flux, nonconservative_flux = surface_integral.surface_flux + @unpack surface_flux = surface_integral index_range = eachnode(dg) @@ -387,28 +386,11 @@ function calc_boundary_flux!(cache, t, boundary_condition, boundary_indexing, k_node = k_node_start for j in eachnode(dg) for i in eachnode(dg) - # Extract solution data from boundary container - u_inner = get_node_vars(boundaries.u, equations, dg, i, j, boundary) - - # Outward-pointing normal direction (not normalized) - normal_direction = get_normal_direction(direction, - contravariant_vectors, - i_node, j_node, k_node, element) - - # Coordinates at boundary node - x = get_node_coords(node_coordinates, equations, dg, - i_node, j_node, k_node, element) - - flux_ = boundary_condition(u_inner, normal_direction, x, t, - surface_flux, equations) - - noncons_ = nonconservative_flux(u_inner, u_inner, normal_direction, - normal_direction, equations) - - # Copy flux to element storage in the correct orientation - for v in eachvariable(equations) - surface_flux_values[v, i, j, direction, element] = flux_[v] + 0.5*noncons_[v] - end + calc_boundary_flux!(surface_flux_values, t, boundary_condition, + mesh, have_nonconservative_terms(equations), + equations, surface_integral, dg, cache, + i_node, j_node, k_node, + i, j, direction, element, boundary) i_node += i_node_step_i j_node += j_node_step_i @@ -421,6 +403,80 @@ function calc_boundary_flux!(cache, t, boundary_condition, boundary_indexing, end end +# Inlined version of the boundary flux calculation along a physical interface. +@inline function calc_boundary_flux!(surface_flux_values, t, boundary_condition, + mesh::Union{P4estMesh{3}, T8codeMesh{3}}, + nonconservative_terms::False, equations, + surface_integral, dg::DG, cache, + i_node, j_node, k_node, + i, j, direction, element, + boundary) + + @unpack boundaries = cache + @unpack surface_flux_values, node_coordinates, contravariant_vectors = cache.elements + @unpack surface_flux = surface_integral + + # Extract solution data from boundary container. + u_inner = get_node_vars(boundaries.u, equations, dg, i, j, boundary) + + # Outward-pointing normal direction (not normalized). + normal_direction = get_normal_direction(direction, + contravariant_vectors, + i_node, j_node, k_node, element) + + # Coordinates at boundary node. + x = get_node_coords(node_coordinates, equations, dg, + i_node, j_node, k_node, element) + + # Flux computation. + flux_ = boundary_condition(u_inner, normal_direction, x, t, + surface_flux, equations) + + # Copy flux to element storage in the correct orientation + for v in eachvariable(equations) + surface_flux_values[v, i, j, direction, element] = flux_[v] + end +end + +# Inlined version of the boundary flux calculation along a physical interface. +@inline function calc_boundary_flux!(surface_flux_values, t, boundary_condition, + mesh::Union{P4estMesh{3}, T8codeMesh{3}}, + nonconservative_terms::True, equations, + surface_integral, dg::DG, cache, + i_node, j_node, k_node, + i, j, direction, element, + boundary) + + @unpack boundaries = cache + @unpack surface_flux_values, node_coordinates, contravariant_vectors = cache.elements + surface_flux, nonconservative_flux = surface_integral.surface_flux + + # Extract solution data from boundary container. + u_inner = get_node_vars(boundaries.u, equations, dg, i, j, boundary) + + # Outward-pointing normal direction (not normalized). + normal_direction = get_normal_direction(direction, + contravariant_vectors, + i_node, j_node, k_node, element) + + # Coordinates at boundary node. + x = get_node_coords(node_coordinates, equations, dg, + i_node, j_node, k_node, element) + + # Flux computation. + flux_ = boundary_condition(u_inner, normal_direction, x, t, + surface_flux, equations) + + # Non-conservative flux computation. + noncons_ = nonconservative_flux(u_inner, u_inner, normal_direction, + normal_direction, equations) + + # Copy flux to element storage in the correct orientation. + for v in eachvariable(equations) + surface_flux_values[v, i, j, direction, element] = flux_[v] + 0.5*noncons_[v] + end +end + function prolong2mortars!(cache, u, mesh::Union{P4estMesh{3}, T8codeMesh{3}}, equations, mortar_l2::LobattoLegendreMortarL2, diff --git a/src/solvers/dgsem_p4est/dg_3d_parallel.jl b/src/solvers/dgsem_p4est/dg_3d_parallel.jl index e504e06d2c4..63aee9517fb 100644 --- a/src/solvers/dgsem_p4est/dg_3d_parallel.jl +++ b/src/solvers/dgsem_p4est/dg_3d_parallel.jl @@ -267,6 +267,40 @@ end end end +# Inlined version of the interface flux computation for conservation laws +@inline function calc_mpi_interface_flux!(surface_flux_values, + mesh::Union{ParallelP4estMesh{3}, + ParallelT8codeMesh{3}}, + nonconservative_terms::True, equations, + surface_integral, dg::DG, cache, + interface_index, normal_direction, + interface_i_node_index, + interface_j_node_index, local_side, + surface_i_node_index, surface_j_node_index, + local_direction_index, local_element_index) + @unpack u = cache.mpi_interfaces + surface_flux, nonconservative_flux = surface_integral.surface_flux + + u_ll, u_rr = get_surface_node_vars(u, equations, dg, + interface_i_node_index, interface_j_node_index, + interface_index) + + if local_side == 1 + flux_ = surface_flux(u_ll, u_rr, normal_direction, equations) + noncons_ = nonconservative_flux(u_ll, u_rr, normal_direction, + normal_direction, equations) + else # local_side == 2 + flux_ = -surface_flux(u_ll, u_rr, -normal_direction, equations) + noncons_ = -nonconservative_flux(u_rr, u_ll, -normal_direction, + -normal_direction, equations) + end + + for v in eachvariable(equations) + surface_flux_values[v, surface_i_node_index, surface_j_node_index, + local_direction_index, local_element_index] = flux_[v] + 0.5 * noncons_[v] + end +end + function prolong2mpimortars!(cache, u, mesh::Union{ParallelP4estMesh{3}, ParallelT8codeMesh{3}}, equations, @@ -460,6 +494,33 @@ end position_index) end +# Inlined version of the mortar flux computation on small elements for conservation laws +@inline function calc_mpi_mortar_flux!(fstar, + mesh::Union{ParallelP4estMesh{3}, + ParallelT8codeMesh{3}}, + nonconservative_terms::True, equations, + surface_integral, dg::DG, cache, + mortar_index, position_index, normal_direction, + i_node_index, j_node_index) + @unpack u = cache.mpi_mortars + surface_flux, nonconservative_flux = surface_integral.surface_flux + + u_ll, u_rr = get_surface_node_vars(u, equations, dg, position_index, i_node_index, + j_node_index, mortar_index) + + flux_ = surface_flux(u_ll, u_rr, normal_direction, equations) + + noncons_ = nonconservative_flux(u_ll, u_rr, normal_direction, + normal_direction, equations) + + flux = flux_ + 0.5 * noncons_ + + # Copy flux to buffer + set_node_vars!(fstar, flux, equations, dg, i_node_index, j_node_index, + position_index) +end + + @inline function mpi_mortar_fluxes_to_elements!(surface_flux_values, mesh::Union{ParallelP4estMesh{3}, ParallelT8codeMesh{3}}, From 21fd96eb29620fcd6496a48c49836e92d8d94c59 Mon Sep 17 00:00:00 2001 From: Johannes Markert Date: Fri, 30 Aug 2024 18:09:12 +0200 Subject: [PATCH 81/81] Backup. --- ...elixir_three_equations_ellitptical_drop.jl | 146 ++++++++++++ src/equations/three_equations_2d.jl | 219 +++++++++++++++--- 2 files changed, 331 insertions(+), 34 deletions(-) create mode 100644 examples/t8code_2d_dgsem/elixir_three_equations_ellitptical_drop.jl diff --git a/examples/t8code_2d_dgsem/elixir_three_equations_ellitptical_drop.jl b/examples/t8code_2d_dgsem/elixir_three_equations_ellitptical_drop.jl new file mode 100644 index 00000000000..e134c556d73 --- /dev/null +++ b/examples/t8code_2d_dgsem/elixir_three_equations_ellitptical_drop.jl @@ -0,0 +1,146 @@ +# The same setup as tree_2d_dgsem/elixir_advection_basic.jl +# to verify the StructuredMesh implementation against TreeMesh + +using OrdinaryDiffEq +using Trixi + +############################################################################### +# semidiscretization of the linear advection equation + +gamma = 1.0 +k0 = 3.2e-5 +rho0 = 1000.0 +# G = 9.81 +G = 0.0 + +equations = ThreeEquations2D(gamma, k0, rho0, G) + +function smootherstep(left, right, x) + # Scale, and clamp x to 0..1 range. + x = clamp((x - left) / (right - left)) + + return x * x * x * (x * (6.0 * x - 15.0) + 10.0) +end + +@inline function clamp(x, lowerlimit = 0.0, upperlimit = 1.0) + if (x < lowerlimit) return lowerlimit end + if (x > upperlimit) return upperlimit end + return x +end + +function initial_condition(x, t, equations::ThreeEquations2D) + + r = Trixi.norm(x) + + width = 0.1 + radius = 1.0 + + s = smootherstep(radius + 0.5*width, radius - 0.5*width, r) + + rho = 1000.0 + v1 = -100.0 * x[1] * s + v2 = 100.0 * x[2] * s + alpha = clamp(s, 1e-3, 1.0) + + # # liquid domain + # if((x[1]^2 + x[2]^2) <= 1) + # rho = 1000.0 + # alpha = 1.0 - 10^(-3) + # v1 = -100.0 * x[1] + # v2 = 100.0 * x[2] + # else + # rho = 1000.0 + # v1 = 0.0 + # v2 = 0.0 + # alpha = 10^(-3) + # end + # phi = x[2] + phi = 0.0 + + # rho = 1000.0 + # v1 = 0.0 + # v2 = 0.0 + # alpha = 1.0 + + return prim2cons(SVector(rho, v1, v2, alpha, phi, 0.0), equations) +end + +# volume_flux = (flux_central, flux_nonconservative_ThreeEquations) +volume_flux = (Trixi.flux_entropy_cons_gamma_one_ThreeEquations, Trixi.flux_non_cons_entropy_cons_gamma_one_ThreeEquations) + +surface_flux = (Trixi.flux_entropy_cons_gamma_one_ThreeEquations, Trixi.flux_non_cons_entropy_cons_gamma_one_ThreeEquations) +# surface_flux = (flux_lax_friedrichs, Trixi.flux_non_cons_entropy_cons_gamma_one_ThreeEquations) + +polydeg = 3 + +basis = LobattoLegendreBasis(polydeg) + +indicator_sc = IndicatorHennemannGassner(equations, basis, + alpha_max=1.0, + alpha_min=0.001, + alpha_smooth=true, + variable=alpha_rho) + +volume_integral = VolumeIntegralShockCapturingHG(indicator_sc; + volume_flux_dg=volume_flux, + volume_flux_fv=surface_flux) + +# volume_integral = VolumeIntegralFluxDifferencing(volume_flux) + +# Create DG solver with polynomial degree = 3 and (local) Lax-Friedrichs/Rusanov flux as surface flux +# solver = DGSEM(basis, surface_flux = surface_flux) +solver = DGSEM(basis, surface_flux, volume_integral) + +coordinates_min = (-3.0, -3.0) # minimum coordinates (min(x), min(y)) +coordinates_max = ( 3.0, 3.0) # maximum coordinates (max(x), max(y)) + +trees_per_dimension = (1, 1) + +initial_refinement_level = 6 + +mesh = P4estMesh(trees_per_dimension, polydeg = polydeg , + coordinates_min = coordinates_min, coordinates_max = coordinates_max, + initial_refinement_level = initial_refinement_level) + +# A semidiscretization collects data structures and functions for the spatial discretization +semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition, solver) + +############################################################################### +# ODE solvers, callbacks etc. + +tspan = (0.0, 0.0076) + +# Create ODE problem with time span from 0.0 to 1.0 +ode = semidiscretize(semi, tspan) + +# At the beginning of the main loop, the SummaryCallback prints a summary of the simulation setup +# and resets the timers +summary_callback = SummaryCallback() + +# The SaveSolutionCallback allows to save the solution to a file in regular intervals +save_solution = SaveSolutionCallback(interval = 20, + solution_variables = cons2cons) + +# The AnalysisCallback allows to analyse the solution in regular intervals and prints the results +analysis_callback = AnalysisCallback(semi, interval = 100) + +# The StepsizeCallback handles the re-calculation of the maximum Δt after each time step +stepsize_callback = StepsizeCallback(cfl = 0.5) + +# Create a CallbackSet to collect all callbacks such that they can be passed to the ODE solver +callbacks = CallbackSet(summary_callback, analysis_callback, stepsize_callback, save_solution) + +############################################################################### +# run the simulation + +# OrdinaryDiffEq's `solve` method evolves the solution in time and executes the passed callbacks +sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false), + dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback + save_everystep = false, callback = callbacks); + +# Print the timer summary +summary_callback() + +# Finalize `T8codeMesh` to make sure MPI related objects in t8code are +# released before `MPI` finalizes. +!isinteractive() && finalize(mesh) diff --git a/src/equations/three_equations_2d.jl b/src/equations/three_equations_2d.jl index 7c47236f091..81861367915 100644 --- a/src/equations/three_equations_2d.jl +++ b/src/equations/three_equations_2d.jl @@ -260,6 +260,23 @@ return SVector(f1, f2, f3, f4, f5, f6) end + @inline function flux_nonconservative_ThreeEquations(u_ll, u_rr, orientation::Integer, + equations::ThreeEquations2D) + v1_ll = u_ll[2] / u_ll[1] + v2_ll = u_ll[3] / u_ll[1] + alpha_rr = u_rr[4] + + z = zero(eltype(u_ll)) + + if orientation == 1 + f = SVector(z, z, z, v1_ll * alpha_rr, z, z) + else + f = SVector(z, z, z, v2_ll * alpha_rr, z, z) + end + + return f + end + # Calculate 1D flux for a single point in the normal direction # Note, this directional vector is not normalized @inline function flux(u, normal_direction::AbstractVector, equations::ThreeEquations2D) @@ -280,40 +297,154 @@ return SVector(f1, f2, f3, f4, f5, f6) end - @inline function flux_nonconservative_ThreeEquations(u_ll, u_rr, orientation::Integer, + @inline function flux_nonconservative_ThreeEquations(u_ll, u_rr, + normal_direction_ll::AbstractVector, + normal_direction_average::AbstractVector, equations::ThreeEquations2D) v1_ll = u_ll[2] / u_ll[1] v2_ll = u_ll[3] / u_ll[1] alpha_rr = u_rr[4] + v_dot_n_ll = v1_ll * normal_direction_ll[1] + v2_ll * normal_direction_ll[2] + z = zero(eltype(u_ll)) - if orientation == 1 - f = SVector(z, z, z, v1_ll * alpha_rr, z, z) - else - f = SVector(z, z, z, v2_ll * alpha_rr, z, z) - end + f = SVector(z, z, z, v_dot_n_ll * alpha_rr, z, z) return f end - @inline function flux_nonconservative_ThreeEquations(u_ll, u_rr, - normal_direction_ll::AbstractVector, - normal_direction_average::AbstractVector, + @inline function flux_entropy_cons_ThreeEquations(u_ll, u_rr, + normal_direction::AbstractVector, equations::ThreeEquations2D) - v1_ll = u_ll[2] / u_ll[1] - v2_ll = u_ll[3] / u_ll[1] - alpha_rr = u_rr[4] + z = zero(eltype(u_ll)) - v_dot_n_ll = v1_ll * normal_direction_ll[1] + v2_ll * normal_direction_ll[2] + rho_ll, v1_ll, v2_ll, alpha_ll, phi_ll, dummy_ll = cons2prim(u_ll, equations) + rho_rr, v1_rr, v2_rr, alpha_rr, phi_rr, dummy_rr = cons2prim(u_rr, equations) + + rho_mean = stolarsky_mean(rho_ll, rho_rr, equations.gamma) + + v1_avg = 0.5f0 * (v1_ll + v1_rr) + v2_avg = 0.5f0 * (v2_ll + v2_rr) + alpha_avg = 0.5f0 * (alpha_ll + alpha_rr) + + v_dot_n_ll = v1_ll * normal_direction[1] + v2_ll * normal_direction[2] + v_dot_n_rr = v1_rr * normal_direction[1] + v2_rr * normal_direction[2] + + v_dot_n_avg = 0.5f0 * (v_dot_n_ll + v_dot_n_rr) + + p_ll = pressure(u_ll, equations) + p_rr = pressure(u_rr, equations) + + p_avg = 0.5f0 * (p_ll + p_rr) + + f1 = alpha_avg * rho_mean * v_dot_n_avg + f2 = f1 * v1_avg + alpha_avg * p_avg * normal_direction[1] + f3 = f1 * v2_avg + alpha_avg * p_avg * normal_direction[2] + f4 = z + f5 = z + + return SVector(f1, f2, f3, f4, f5, z) + end + + @inline function flux_non_cons_entropy_cons_ThreeEquations(u_ll, u_rr, + normal_direction_ll::AbstractVector, + normal_direction_average::AbstractVector, + equations::ThreeEquations2D) z = zero(eltype(u_ll)) - f = SVector(z, z, z, v_dot_n_ll * alpha_rr, z, z) + rho_ll, v1_ll, v2_ll, alpha_ll, phi_ll, dummy_ll = cons2prim(u_ll, equations) + rho_rr, v1_rr, v2_rr, alpha_rr, phi_rr, dummy_rr = cons2prim(u_rr, equations) - return f + phi_jump = phi_rr - phi_ll + alpha_jump = alpha_rr - alpha_ll + + alpha_avg = 0.5f0 * (alpha_ll + alpha_rr) + + rho_mean = stolarsky_mean(rho_ll, rho_rr, equations.gamma) + # rho_mean = 0.5f0 * (rho_ll + rho_rr) + + v_dot_n_ll = v1_ll * normal_direction_average[1] + v2_ll * normal_direction_average[2] + v_dot_n_rr = v1_rr * normal_direction_average[1] + v2_rr * normal_direction_average[2] + + v_dot_n_avg = 0.5f0 * (v_dot_n_ll + v_dot_n_rr) + + f1 = z + f2 = phi_jump * alpha_avg * rho_mean * normal_direction_average[1] + f3 = phi_jump * alpha_avg * rho_mean * normal_direction_average[2] + # f4 = alpha_jump * v_dot_n_avg + f4 = alpha_jump * v_dot_n_ll + f5 = z + + return SVector(f1, f2, f3, f4, f5, z) end + ## @inline function flux_entropy_cons_gamma_one_ThreeEquations(u_ll, u_rr, + ## normal_direction::AbstractVector, + ## equations::ThreeEquations2D) + ## z = zero(eltype(u_ll)) + + ## rho_ll, v1_ll, v2_ll, alpha_ll, phi_ll, dummy_ll = cons2prim(u_ll, equations) + ## rho_rr, v1_rr, v2_rr, alpha_rr, phi_rr, dummy_rr = cons2prim(u_rr, equations) + + ## rho_mean = ln_mean(rho_ll, rho_rr) + + ## v1_avg = 0.5f0 * (v1_ll + v1_rr) + ## v2_avg = 0.5f0 * (v2_ll + v2_rr) + + ## alpha_avg = 0.5f0 * (alpha_ll + alpha_rr) + + ## v_dot_n_ll = v1_ll * normal_direction[1] + v2_ll * normal_direction[2] + ## v_dot_n_rr = v1_rr * normal_direction[1] + v2_rr * normal_direction[2] + + ## v_dot_n_avg = 0.5f0 * (v_dot_n_ll + v_dot_n_rr) + + ## p_ll = pressure(u_ll, equations) + ## p_rr = pressure(u_rr, equations) + + ## p_avg = 0.5f0 * (p_ll + p_rr) + + ## f1 = alpha_avg * rho_mean * v_dot_n_avg + ## f2 = f1 * v1_avg + alpha_avg * p_avg * normal_direction[1] + ## f3 = f1 * v2_avg + alpha_avg * p_avg * normal_direction[2] + ## f4 = z + ## f5 = z + + ## return SVector(f1, f2, f3, f4, f5, z) + ## end + + ## @inline function flux_non_cons_entropy_cons_gamma_one_ThreeEquations(u_ll, u_rr, + ## normal_direction_ll::AbstractVector, + ## normal_direction_average::AbstractVector, + ## equations::ThreeEquations2D) + ## z = zero(eltype(u_ll)) + + ## rho_ll, v1_ll, v2_ll, alpha_ll, phi_ll, dummy_ll = cons2prim(u_ll, equations) + ## rho_rr, v1_rr, v2_rr, alpha_rr, phi_rr, dummy_rr = cons2prim(u_rr, equations) + + ## phi_jump = phi_rr - phi_ll + ## alpha_jump = alpha_rr - alpha_ll + + ## alpha_avg = 0.5f0 * (alpha_ll + alpha_rr) + + ## rho_mean = ln_mean(rho_ll, rho_rr) + ## # rho_mean = 0.5f0 * (rho_ll + rho_rr) + + ## v_dot_n_ll = v1_ll * normal_direction_average[1] + v2_ll * normal_direction_average[2] + ## v_dot_n_rr = v1_rr * normal_direction_average[1] + v2_rr * normal_direction_average[2] + + ## v_dot_n_avg = 0.5f0 * (v_dot_n_ll + v_dot_n_rr) + + ## f1 = z + ## f2 = phi_jump * alpha_avg * rho_mean * normal_direction_average[1] + ## f3 = phi_jump * alpha_avg * rho_mean * normal_direction_average[2] + ## f4 = alpha_jump * v_dot_n_avg + ## f5 = z + + ## return SVector(f1, f2, f3, f4, f5, z) + ## end + @inline function flux_nonconservative_ThreeEquations_well(u_ll, u_rr, orientation::Integer, equations::ThreeEquations2D) v1_ll = u_ll[2] / u_ll[1] @@ -351,9 +482,9 @@ v_dot_n_ll = v1_ll * normal_direction_ll[1] + v2_ll * normal_direction_ll[2] # Non well-balanced force contribution. - force = -u_ll[1] * phi_rr + force = u_ll[1] * phi_rr - # Well-balanced force contribution. + # # Well-balanced force contribution. # force = -u_ll[1] * (equations.k0/equations.rho_0) * # exp(-equations.rho_0/equations.k0 * phi_ll) * # exp( equations.rho_0/equations.k0 * phi_rr) @@ -472,24 +603,17 @@ # Convert conservative variables to primitive @inline function cons2entropy(u, equations::ThreeEquations2D) - alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi, dummy = u - - rho = alpha_rho / alpha - v1 = alpha_rho_v1 / alpha_rho - v2 = alpha_rho_v2 / alpha_rho - - phi = max(1e-6, max(max_abs_speeds(u, equations)...) ) - - # vn = sqrt(v1*v1 + v2*v2) - # vn = log10(sqrt(alpha_rho_v1^2 + alpha_rho_v2^2)) - vn = log10(max(1e-6, sqrt(v1^2 + v2^2))) - - alpha_log = log10(max(1e-6, alpha)) + rho, v1, v2, alpha, phi, dummy = cons2prim(u, equations) p = pressure(u, equations) - # return SVector(alpha_rho, vn, log10(alpha), alpha, phi) - # return SVector(alpha_rho, vn, alpha_log, alpha, phi) - return SVector(alpha_rho, p, alpha_log, alpha, phi, dummy) + w1 = internal_energy_density(u, equations) - 0.5*(v1^2 + v2^2) + p/rho + phi + w2 = v1 + w3 = v2 + w4 = -p + w5 = alpha*rho + w6 = 0.0 + + return SVector(w1, w2, w3, w4, w5, w6) end # Convert primitive to conservative variables @@ -524,10 +648,37 @@ @inline function pressure(u, equations::ThreeEquations2D) alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi, dummy = u - # return alpha * equations.k0 * ((alpha_rho / alpha / equations.rho_0)^(equations.gamma) - 1) return equations.k0 * ((alpha_rho / alpha / equations.rho_0)^(equations.gamma) - 1) end + @inline function internal_energy_density(u, equations::ThreeEquations2D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi, dummy = u + return alpha*equations.k0/alpha_rho * ((alpha_rho / alpha / equations.rho_0)^(equations.gamma)/(equations.gamma-1) + 1) + end + + @inline function internal_energy(u, equations::ThreeEquations2D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi, dummy = u + return alpha_rho * internal_energy_density(u, equations) + end + + @inline function kinetic_energy(u, equations::ThreeEquations2D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi, dummy = u + return 0.5*(alpha_rho_v1^2 + alpha_rho_v2^2)/alpha_rho + end + + @inline function potential_energy(u, equations::ThreeEquations2D) + alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi, dummy = u + return alpha_rho*phi + end + + @inline function total_energy(u, equations::ThreeEquations2D) + return kinetic_energy(u, equations) + internal_energy(u, equations) + potential_energy(u, equations) + end + + @inline function entropy(u, equations::ThreeEquations2D) + return total_energy(u, equations) + end + @inline function density_pressure(u, equations::ThreeEquations2D) alpha_rho, alpha_rho_v1, alpha_rho_v2, alpha, phi, dummy = u rho = alpha_rho / alpha