diff --git a/src/auxiliary/auxiliary.jl b/src/auxiliary/auxiliary.jl index e56cd399f7a..28e4c47d339 100644 --- a/src/auxiliary/auxiliary.jl +++ b/src/auxiliary/auxiliary.jl @@ -82,6 +82,26 @@ end return ncalls_first end +""" + @trixi_timeit_ext backend timer() "some label" expression + +This macro is an extension of [`@trixi_timeit`](@ref) that also synchronizes the given `backend` after executing the given `expression`. +This is useful to get accurate timing measurements for GPU backends, where the execution of kernels is asynchronous. +The synchronization ensures that all GPU operations are completed before the timer is stopped. + +See also [`@trixi_timeit`](@ref). +""" +macro trixi_timeit_ext(backend, timer_output, label, expr) + expr = quote + local val = $(esc(expr)) + if $(esc(backend)) !== nothing && $(TrixiBase).timeit_debug_enabled() + $(KernelAbstractions.synchronize)($(esc(backend))) + end + val + end + return :(@trixi_timeit($(esc(timer_output)), $(esc(label)), $(expr))) +end + """ examples_dir() diff --git a/src/solvers/dgsem_tree/dg_2d.jl b/src/solvers/dgsem_tree/dg_2d.jl index 55002f3da25..739706d209b 100644 --- a/src/solvers/dgsem_tree/dg_2d.jl +++ b/src/solvers/dgsem_tree/dg_2d.jl @@ -113,63 +113,66 @@ function rhs!(du, u, t, backend = trixi_backend(u) # Reset du - @trixi_timeit timer() "reset ∂u/∂t" set_zero!(du, dg, cache) + @trixi_timeit_ext backend timer() "reset ∂u/∂t" begin + set_zero!(du, dg, cache) + end # Calculate volume integral - @trixi_timeit timer() "volume integral" begin + @trixi_timeit_ext backend timer() "volume integral" begin calc_volume_integral!(backend, du, u, mesh, have_nonconservative_terms(equations), equations, dg.volume_integral, dg, cache) end # Prolong solution to interfaces - @trixi_timeit timer() "prolong2interfaces" begin + @trixi_timeit_ext backend timer() "prolong2interfaces" begin prolong2interfaces!(backend, cache, u, mesh, equations, dg) end # Calculate interface fluxes - @trixi_timeit timer() "interface flux" begin + @trixi_timeit_ext backend timer() "interface flux" begin calc_interface_flux!(backend, cache.elements.surface_flux_values, mesh, have_nonconservative_terms(equations), equations, dg.surface_integral, dg, cache) end # Prolong solution to boundaries - @trixi_timeit timer() "prolong2boundaries" begin + @trixi_timeit_ext backend timer() "prolong2boundaries" begin prolong2boundaries!(cache, u, mesh, equations, dg) end # Calculate boundary fluxes - @trixi_timeit timer() "boundary flux" begin + @trixi_timeit_ext backend timer() "boundary flux" begin calc_boundary_flux!(cache, t, boundary_conditions, mesh, equations, dg.surface_integral, dg) end # Prolong solution to mortars - @trixi_timeit timer() "prolong2mortars" begin + @trixi_timeit_ext backend timer() "prolong2mortars" begin prolong2mortars!(cache, u, mesh, equations, dg.mortar, dg) end # Calculate mortar fluxes - @trixi_timeit timer() "mortar flux" begin + @trixi_timeit_ext backend timer() "mortar flux" begin calc_mortar_flux!(cache.elements.surface_flux_values, mesh, have_nonconservative_terms(equations), equations, dg.mortar, dg.surface_integral, dg, cache) end # Calculate surface integrals - @trixi_timeit timer() "surface integral" begin + @trixi_timeit_ext backend timer() "surface integral" begin calc_surface_integral!(backend, du, u, mesh, equations, dg.surface_integral, dg, cache) end # Apply Jacobian from mapping to reference element - @trixi_timeit timer() "Jacobian" apply_jacobian!(backend, du, mesh, equations, dg, - cache) + @trixi_timeit_ext backend timer() "Jacobian" begin + apply_jacobian!(backend, du, mesh, equations, dg, cache) + end # Calculate source terms - @trixi_timeit timer() "source terms" begin + @trixi_timeit_ext backend timer() "source terms" begin calc_sources!(du, u, t, source_terms, equations, dg, cache) end