Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor tests based on new GPU cache #60

Merged
merged 7 commits into from
Sep 30, 2024
Merged

Refactor tests based on new GPU cache #60

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
448bf37
Start
huiyuxie Sep 27, 2024
5cff202
Minor fix
huiyuxie Sep 27, 2024
f10592b
Add tests
huiyuxie Sep 28, 2024
5f2ddd9
Fix errors from upstream
huiyuxie Sep 28, 2024
3c9ac74
Add tests
huiyuxie Sep 29, 2024
27342e6
Add tests
huiyuxie Sep 30, 2024
b722368
Add tests
huiyuxie Sep 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/TrixiCUDA.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ using Trixi: AbstractEquations, AbstractContainer,
L2MortarContainer2D, L2MortarContainer3D,
True, False,
TreeMesh, DGSEM,
BoundaryConditionPeriodic, SemidiscretizationHyperbolic,
SemidiscretizationHyperbolic,
BoundaryConditionPeriodic, BoundaryConditionDirichlet,
VolumeIntegralWeakForm, VolumeIntegralFluxDifferencing, VolumeIntegralShockCapturingHG,
LobattoLegendreMortarL2,
flux, ntuple, nvariables, nnodes, nelements, nmortars,
Expand Down
2 changes: 1 addition & 1 deletion src/solvers/common.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

# Copy data from CPU to GPU
function copy_to_gpu!(du::PtrArray, u::PtrArray)
du = CuArray{Float64}(zero(du))
du = CUDA.zeros(Float64, size(du)...)
u = CuArray{Float64}(u)

return (du, u)
Expand Down
1 change: 0 additions & 1 deletion src/solvers/containers_3d.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ mutable struct ElementContainerGPU3D{RealT <: Real, uEltype <: Real} <: Abstract
surface_flux_values::CuArray{uEltype, 5}
cell_ids::CuArray{Int, 1}

# Inner constructor
# Inner constructor
function ElementContainerGPU3D{RealT, uEltype}(dims_inverse_jacobian::NTuple{1, Int},
dims_node_coordinates::NTuple{5, Int},
Expand Down
29 changes: 15 additions & 14 deletions src/solvers/dg_1d.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -703,8 +703,8 @@ function cuda_volume_integral!(du, u, mesh::TreeMesh{1}, nonconservative_terms::
# For `Float32`, this gives 1.1920929f-5
atol = 1.8189894035458565e-12 # see also `pure_and_blended_element_ids!` in Trixi.jl

element_ids_dg = zero(CuArray{Int64}(undef, length(alpha)))
element_ids_dgfv = zero(CuArray{Int64}(undef, length(alpha)))
element_ids_dg = CUDA.zeros(Int, length(alpha))
element_ids_dgfv = CUDA.zeros(Int, length(alpha))

pure_blended_element_count_kernel = @cuda launch=false pure_blended_element_count_kernel!(element_ids_dg,
element_ids_dgfv,
Expand Down Expand Up @@ -773,8 +773,8 @@ function cuda_volume_integral!(du, u, mesh::TreeMesh{1}, nonconservative_terms::
# For `Float32`, this gives 1.1920929f-5
atol = 1.8189894035458565e-12 # see also `pure_and_blended_element_ids!` in Trixi.jl

element_ids_dg = zero(CuArray{Int64}(undef, length(alpha)))
element_ids_dgfv = zero(CuArray{Int64}(undef, length(alpha)))
element_ids_dg = CUDA.zeros(Int, length(alpha))
element_ids_dgfv = CUDA.zeros(Int, length(alpha))

pure_blended_element_count_kernel = @cuda launch=false pure_blended_element_count_kernel!(element_ids_dg,
element_ids_dgfv,
Expand Down Expand Up @@ -966,14 +966,15 @@ function cuda_boundary_flux!(t, mesh::TreeMesh{1}, boundary_conditions::NamedTup
lasts = zero(n_boundaries_per_direction)
firsts = zero(n_boundaries_per_direction)

# May introduce kernel launching overhead
last_first_indices_kernel = @cuda launch=false last_first_indices_kernel!(lasts, firsts,
n_boundaries_per_direction)
last_first_indices_kernel(lasts, firsts, n_boundaries_per_direction;
configurator_1d(last_first_indices_kernel, lasts)...)

lasts, firsts = Array(lasts), Array(firsts)
boundary_arr = CuArray{Int64}(firsts[1]:lasts[2])
indices_arr = CuArray{Int64}([firsts[1], firsts[2]])
indices_arr = firsts
boundary_arr = CuArray{Int}(Array(firsts)[1]:Array(lasts)[end])

boundary_conditions_callable = replace_boundary_conditions(boundary_conditions)

boundary_flux_kernel = @cuda launch=false boundary_flux_kernel!(surface_flux_values,
Expand Down Expand Up @@ -1009,30 +1010,30 @@ function cuda_boundary_flux!(t, mesh::TreeMesh{1}, boundary_conditions::NamedTup
lasts = zero(n_boundaries_per_direction)
firsts = zero(n_boundaries_per_direction)

# May introduce kernel launching overhead
last_first_indices_kernel = @cuda launch=false last_first_indices_kernel!(lasts, firsts,
n_boundaries_per_direction)
last_first_indices_kernel(lasts, firsts, n_boundaries_per_direction;
configurator_1d(last_first_indices_kernel, lasts)...)

lasts, firsts = Array(lasts), Array(firsts)
boundary_arr = CuArray{Int64}(firsts[1]:lasts[2])
indices_arr = CuArray{Int64}([firsts[1], firsts[2]])
indices_arr = firsts
boundary_arr = CuArray{Int}(Array(firsts)[1]:Array(lasts)[end])

# Replace with callable functions (not necessary here)
# boundary_conditions_callable = replace_boundary_conditions(boundary_conditions)
boundary_conditions_callable = replace_boundary_conditions(boundary_conditions)

boundary_flux_kernel = @cuda launch=false boundary_flux_kernel!(surface_flux_values,
boundaries_u, node_coordinates,
t, boundary_arr, indices_arr,
neighbor_ids, neighbor_sides,
orientations,
boundary_conditions,
boundary_conditions_callable,
equations,
surface_flux,
nonconservative_flux)
boundary_flux_kernel(surface_flux_values, boundaries_u, node_coordinates, t, boundary_arr,
indices_arr, neighbor_ids, neighbor_sides, orientations,
boundary_conditions, equations, surface_flux, nonconservative_flux;
boundary_conditions_callable, equations, surface_flux,
nonconservative_flux;
configurator_1d(boundary_flux_kernel, boundary_arr)...)

return nothing
Expand Down
33 changes: 16 additions & 17 deletions src/solvers/dg_2d.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1125,8 +1125,8 @@ function cuda_volume_integral!(du, u, mesh::TreeMesh{2}, nonconservative_terms::
# For `Float32`, this gives 1.1920929f-5
atol = 1.8189894035458565e-12 # see also `pure_and_blended_element_ids!` in Trixi.jl

element_ids_dg = zero(CuArray{Int64}(undef, length(alpha)))
element_ids_dgfv = zero(CuArray{Int64}(undef, length(alpha)))
element_ids_dg = CUDA.zeros(Int, length(alpha))
element_ids_dgfv = CUDA.zeros(Int, length(alpha))

pure_blended_element_count_kernel = @cuda launch=false pure_blended_element_count_kernel!(element_ids_dg,
element_ids_dgfv,
Expand Down Expand Up @@ -1209,8 +1209,8 @@ function cuda_volume_integral!(du, u, mesh::TreeMesh{2}, nonconservative_terms::
# For `Float32`, this gives 1.1920929f-5
atol = 1.8189894035458565e-12 # see also `pure_and_blended_element_ids!` in Trixi.jl

element_ids_dg = zero(CuArray{Int64}(undef, length(alpha)))
element_ids_dgfv = zero(CuArray{Int64}(undef, length(alpha)))
element_ids_dg = CUDA.zeros(Int, length(alpha))
element_ids_dgfv = CUDA.zeros(Int, length(alpha))

pure_blended_element_count_kernel = @cuda launch=false pure_blended_element_count_kernel!(element_ids_dg,
element_ids_dgfv,
Expand Down Expand Up @@ -1440,16 +1440,16 @@ function cuda_boundary_flux!(t, mesh::TreeMesh{2}, boundary_conditions::NamedTup
lasts = zero(n_boundaries_per_direction)
firsts = zero(n_boundaries_per_direction)

# May introduce kernel launching overhead
last_first_indices_kernel = @cuda launch=false last_first_indices_kernel!(lasts, firsts,
n_boundaries_per_direction)
last_first_indices_kernel(lasts, firsts, n_boundaries_per_direction;
configurator_1d(last_first_indices_kernel, lasts)...)

lasts, firsts = Array(lasts), Array(firsts)
boundary_arr = CuArray{Int64}(firsts[1]:lasts[4])
indices_arr = CuArray{Int64}([firsts[1], firsts[2], firsts[3], firsts[4]])
boundary_conditions_callable = replace_boundary_conditions(boundary_conditions)
indices_arr = firsts
boundary_arr = CuArray{Int}(Array(firsts)[1]:Array(lasts)[end])

boundary_conditions_callable = replace_boundary_conditions(boundary_conditions)
size_arr = CuArray{Float64}(undef, size(surface_flux_values, 2), length(boundary_arr))

boundary_flux_kernel = @cuda launch=false boundary_flux_kernel!(surface_flux_values,
Expand Down Expand Up @@ -1485,32 +1485,31 @@ function cuda_boundary_flux!(t, mesh::TreeMesh{2}, boundary_conditions::NamedTup
lasts = zero(n_boundaries_per_direction)
firsts = zero(n_boundaries_per_direction)

# May introduce kernel launching overhead
last_first_indices_kernel = @cuda launch=false last_first_indices_kernel!(lasts, firsts,
n_boundaries_per_direction)
last_first_indices_kernel(lasts, firsts, n_boundaries_per_direction;
configurator_1d(last_first_indices_kernel, lasts)...)

lasts, firsts = Array(lasts), Array(firsts)
boundary_arr = CuArray{Int64}(firsts[1]:lasts[4])
indices_arr = CuArray{Int64}([firsts[1], firsts[2], firsts[3], firsts[4]])

# Replace with callable functions (not necessary here)
# boundary_conditions_callable = replace_boundary_conditions(boundary_conditions)
indices_arr = firsts
boundary_arr = CuArray{Int}(Array(firsts)[1]:Array(lasts)[end])

boundary_conditions_callable = replace_boundary_conditions(boundary_conditions)
size_arr = CuArray{Float64}(undef, size(surface_flux_values, 2), length(boundary_arr))

boundary_flux_kernel = @cuda launch=false boundary_flux_kernel!(surface_flux_values,
boundaries_u, node_coordinates,
t, boundary_arr, indices_arr,
neighbor_ids, neighbor_sides,
orientations,
boundary_conditions,
boundary_conditions_callable,
equations,
surface_flux,
nonconservative_flux)
boundary_flux_kernel(surface_flux_values, boundaries_u, node_coordinates, t, boundary_arr,
indices_arr, neighbor_ids, neighbor_sides, orientations,
boundary_conditions, equations, surface_flux, nonconservative_flux;
boundary_conditions_callable, equations, surface_flux,
nonconservative_flux;
configurator_2d(boundary_flux_kernel, size_arr)...)

return nothing
Expand Down Expand Up @@ -1577,7 +1576,7 @@ function cuda_mortar_flux!(mesh::TreeMesh{2}, cache_mortars::True, nonconservati
large_sides = cache.mortars.large_sides
orientations = cache.mortars.orientations

#
# The original CPU arrays hold NaNs
u_upper = cache.mortars.u_upper
u_lower = cache.mortars.u_lower
reverse_upper = CuArray{Float64}(dg.mortar.reverse_upper)
Expand Down
Loading
Loading