Allow Hydro to use its own EB setting instead of inheriting from AMReX

CMakeLists.txt

@@ -26,6 +26,7 @@ endif()
 set(HYDRO_SPACEDIM 3 CACHE STRING "Dimension of AMReX build: <2,3>")
 option( HYDRO_PROJECTIONS  "Enable projections" YES)
 option( HYDRO_EB     "Enable Embedded-Boundary support" YES)
+option( HYDRO_NO_EB  "Disable EB support even if AMReX has EB support" NO)
 option( HYDRO_OMP    "Enable OpenMP" NO )
 option( HYDRO_MPI    "Enable MPI"   YES )
 option( HYDRO_FFT    "Enable FFT"   NO )
@@ -112,10 +113,16 @@ if (HYDRO_PROJECTIONS)
    add_subdirectory(Projections)
 endif ()
 
+if (HYDRO_EB AND HYDRO_NO_EB)
+    message(FATAL_ERROR "HYDRO_EB and HYDRO_NO_EB cannot be enabled at the same time.")
+endif()
 if (HYDRO_EB)
    add_subdirectory(EBMOL)
    add_subdirectory(EBGodunov)
 endif ()
+if (HYDRO_NO_EB)
+  target_compile_definitions(amrex_hydro PUBLIC HYDRO_NO_EB)
+endif()
 
 if (HYDRO_GPU_BACKEND STREQUAL "CUDA")
    setup_target_for_cuda_compilation(amrex_hydro)

EBGodunov/Make.package

@@ -11,3 +11,7 @@ ifeq ($(AMREX_SPACEDIM),3)
      CEXE_headers += hydro_ebgodunov_corner_couple.H
 endif
 CEXE_headers += hydro_ebgodunov_transverse_$(DIM)D_K.H
+
+ifeq ($(NO_EB), TRUE)
+  DEFINES += -DHYDRO_NO_EB
+endif

EBMOL/Make.package

@@ -8,4 +8,8 @@ CEXE_sources += hydro_ebmol_edge_state.cpp
 CEXE_sources += hydro_ebmol_extrap_vel_to_faces.cpp
 CEXE_sources += hydro_ebmol_extrap_vel_to_faces_box.cpp
 
+ifeq ($(NO_EB), TRUE)
+  DEFINES += -DHYDRO_NO_EB
+endif
+
 endif

EBMOL/hydro_ebmol_extrap_vel_to_faces.cpp

@@ -28,7 +28,7 @@ EBMOL::ExtrapVelToFaces ( const MultiFab&  a_vel,
 {
     BL_PROFILE("EBMOL::ExtrapVelToFaces");
 
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
     AMREX_ASSERT(a_vel.hasEBFabFactory());
     auto const& fact  = dynamic_cast<EBFArrayBoxFactory const&>(a_vel.Factory());
     auto const& flags = fact.getMultiEBCellFlagFab();
@@ -52,7 +52,7 @@ EBMOL::ExtrapVelToFaces ( const MultiFab&  a_vel,
 
             Array4<Real const> const& vcc = a_vel.const_array(mfi);
 
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
             Box const& bx = mfi.tilebox();
             EBCellFlagFab const& flagfab = flags[mfi];
             Array4<EBCellFlag const> const& flagarr = flagfab.const_array();

Projections/Make.package

@@ -8,3 +8,7 @@ ifeq ($(USE_FFT),TRUE)
   CEXE_sources += hydro_FFTMacProjector.cpp
   CEXE_headers += hydro_FFTMacProjector.H
 endif
+
+ifeq ($(NO_EB), TRUE)
+  DEFINES += -DHYDRO_NO_EB
+endif

Projections/hydro_MacProjector.H

@@ -6,7 +6,7 @@
 #include <AMReX_MLPoisson.H>
 #include <AMReX_MLABecLaplacian.H>
 
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
 #include <AMReX_MLEBABecLap.H>
 #endif
 
@@ -25,7 +25,7 @@ class MacProjector
 public:
     MacProjector(
         const amrex::Vector<amrex::Geometry>& a_geom,
-#ifndef AMREX_USE_EB
+#if !defined(AMREX_USE_EB) || defined(HYDRO_NO_EB)
         amrex::MLMG::Location a_umac_loc = amrex::MLMG::Location::FaceCenter,
         amrex::MLMG::Location a_beta_loc = amrex::MLMG::Location::FaceCenter,
         amrex::MLMG::Location a_phi_loc = amrex::MLMG::Location::CellCenter,
@@ -63,7 +63,7 @@ public:
      : MacProjector(a_umac, a_umac_loc, a_beta, a_beta_loc, a_phi_loc,
                     a_geom, a_lpinfo, a_divu, amrex::MLMG::Location::CellCenter) {}
 
-#ifndef AMREX_USE_EB
+#if !defined(AMREX_USE_EB) || defined(HYDRO_NO_EB)
     MacProjector (const amrex::Vector<amrex::Array<amrex::MultiFab*,AMREX_SPACEDIM> >& a_umac,
                   const amrex::Vector<amrex::Array<amrex::MultiFab const*,AMREX_SPACEDIM> >& a_beta,
                   const amrex::Vector<amrex::Geometry>& a_geom,
@@ -101,7 +101,7 @@ public:
     //! Update Bcoeff and if Robin BC also reset scalars in order to reuse the linear operator
     void updateCoeffs (const amrex::Vector<amrex::Array<amrex::MultiFab const*,AMREX_SPACEDIM> >&);
 
-#ifndef AMREX_USE_EB
+#if !defined(AMREX_USE_EB) || defined(HYDRO_NO_EB)
     void initProjector (amrex::Vector<amrex::BoxArray> const& a_grids,
                         amrex::Vector<amrex::DistributionMapping> const& a_dmap,
                         amrex::LPInfo a_lpinfo, amrex::Real a_const_beta,
@@ -116,7 +116,7 @@ public:
     //! Set div(U)
     void setDivU(const amrex::Vector<amrex::MultiFab const*>&);
 
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
     void setEBInflowVelocity (int amrlev, const amrex::MultiFab& eb_vel);
 #endif
 
@@ -171,7 +171,7 @@ private:
 
     std::unique_ptr<amrex::MLPoisson> m_poisson;
     std::unique_ptr<amrex::MLABecLaplacian> m_abeclap;
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
     std::unique_ptr<amrex::MLEBABecLap> m_eb_abeclap;
     amrex::Vector<amrex::EBFArrayBoxFactory const*> m_eb_factory;
     amrex::Vector<std::unique_ptr<amrex::MultiFab> > m_eb_vel;

Projections/hydro_MacProjector.cpp

@@ -1,4 +1,4 @@
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
 #include <AMReX_EBMultiFabUtil.H>
 #endif
 
@@ -76,7 +76,7 @@ void MacProjector::initProjector (
     }
 #endif
 
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
     bool has_eb = a_beta[0][0]->hasEBFabFactory();
     if (has_eb) {
         AMREX_ALWAYS_ASSERT_WITH_MESSAGE(false == m_use_mlhypre, "mlhypre does not work with EB");
@@ -191,7 +191,7 @@ void MacProjector::updateCoeffs (
         "MacProjector::updateCoeffs: should not be called for constant beta");
 
     const auto nlevs = int(a_beta.size());
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
     const bool has_eb = a_beta[0][0]->hasEBFabFactory();
     if (has_eb) {
         if (m_has_robin) {
@@ -229,7 +229,7 @@ void MacProjector::setDivU(const Vector<MultiFab const*>& a_divu)
     for (int ilev = 0, N = int(a_divu.size()); ilev < N; ++ilev) {
         if (a_divu[ilev]) {
             if (!m_divu[ilev].ok()) {
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
                 m_divu[ilev].define(
                     a_divu[ilev]->boxArray(),
                     a_divu[ilev]->DistributionMap(),
@@ -301,7 +301,7 @@ MacProjector::project (Real reltol, Real atol)
         for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
             u[idim] = m_umac[ilev][idim];
         }
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
         if (m_umac_loc != MLMG::Location::FaceCentroid)
         {
             for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
@@ -403,7 +403,7 @@ MacProjector::project (Real reltol, Real atol)
             } else {
                 MultiFab::Add(*m_umac[ilev][idim], m_fluxes[ilev][idim], 0, 0, 1, 0);
             }
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
             EB_set_covered_faces(m_umac[ilev], 0.0);
 #endif
         }
@@ -538,7 +538,7 @@ MacProjector::averageDownVelocity ()
 
         IntVect rr  = m_geom[lev].Domain().size() / m_geom[lev-1].Domain().size();
 
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
         EB_average_down_faces(GetArrOfConstPtrs(m_umac[lev]),
                               m_umac[lev-1],
                               rr, m_geom[lev-1]);
@@ -550,7 +550,7 @@ MacProjector::averageDownVelocity ()
     }
 }
 
-#ifndef AMREX_USE_EB
+#if !defined(AMREX_USE_EB) || defined(HYDRO_NO_EB)
 void MacProjector::initProjector (Vector<BoxArray> const& a_grids,
                                   Vector<DistributionMapping> const& a_dmap,
                                   LPInfo a_lpinfo, Real const a_const_beta,
@@ -657,7 +657,7 @@ void MacProjector::updateBeta (Real a_const_beta)
 
 #endif
 
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
 void MacProjector::setEBInflowVelocity (int amrlev, const MultiFab& eb_vel)
 {
 

Projections/hydro_NodalProjector.H

@@ -130,7 +130,7 @@ private:
     amrex::Vector<amrex::Geometry>               m_geom;
 
     // EB factory if any
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
     amrex::Vector<amrex::EBFArrayBoxFactory const *>  m_ebfactory;
 #endif
 

Projections/hydro_NodalProjector.cpp

@@ -76,7 +76,7 @@ void NodalProjector::define (LPInfo const& a_lpinfo)
     m_rhs.resize(nlevs);
 
 
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
     bool has_eb = m_vel[0] -> hasEBFabFactory();
     if (has_eb)
     {
@@ -123,7 +123,7 @@ void NodalProjector::define (LPInfo const& a_lpinfo)
     // Setup linear operator
     //
     if (m_sigma.empty()) {
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
         m_linop = std::make_unique<MLNodeLaplacian>(m_geom, ba, dm, a_lpinfo, m_ebfactory,
                                                     m_const_sigma);
 #else
@@ -132,7 +132,7 @@ void NodalProjector::define (LPInfo const& a_lpinfo)
                                                     m_const_sigma);
 #endif
     } else {
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
         m_linop = std::make_unique<MLNodeLaplacian>(m_geom, ba, dm, a_lpinfo, m_ebfactory);
 #else
         m_linop = std::make_unique<MLNodeLaplacian>(m_geom, ba, dm, a_lpinfo);
@@ -559,7 +559,7 @@ NodalProjector::averageDown (const amrex::Vector<amrex::MultiFab*>& a_var)
     {
         IntVect rr   = m_geom[lev+1].Domain().size() / m_geom[lev].Domain().size();
 
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
         const auto ebf = dynamic_cast<EBFArrayBoxFactory const&>(a_var[lev+1]->Factory());
 
         amrex::MultiFab volume(a_var[lev+1]->boxArray(),a_var[lev+1]->DistributionMap(),1,0);

Utils/Make.package

@@ -6,3 +6,7 @@ CEXE_headers += hydro_bcs_K.H
 CEXE_headers += hydro_utils.H
 
 CEXE_headers += hydro_constants.H
+
+ifeq ($(NO_EB), TRUE)
+  DEFINES += -DHYDRO_NO_EB
+endif

Utils/hydro_compute_edgestate_and_flux.cpp

@@ -7,7 +7,7 @@
 #include <hydro_mol.H>
 #include <hydro_utils.H>
 
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
 #include <hydro_ebgodunov.H>
 #include <hydro_ebmol.H>
 #endif
@@ -18,7 +18,7 @@ namespace {
     // Limit this function to this file
     void
     ComputeEdgeState (Box const& bx, int ncomp,
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
                       MFIter& mfi,
 #else
                       MFIter& /*mfi*/,
@@ -37,7 +37,7 @@ namespace {
                       Vector<BCRec> const& h_bcrec,
                       const BCRec* d_bcrec,
                       int const* iconserv,
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
                       const EBFArrayBoxFactory& ebfact,
                       Array4<Real const> const& values_on_eb_inflow,
                       bool regular,
@@ -49,7 +49,7 @@ namespace {
                       bool allow_inflow_on_outflow,
                       amrex::Array4<int const> const& bc_arr)
     {
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
         if (!regular)
         {
             EBCellFlagFab const& flagfab = ebfact.getMultiEBCellFlagFab()[mfi];
@@ -145,7 +145,7 @@ namespace {
     }
 } // namespace
 
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
 void
 HydroUtils::ComputeFluxesOnBoxFromState (Box const& bx, int ncomp, MFIter& mfi,
                                          Array4<Real const> const& q,
@@ -251,7 +251,7 @@ HydroUtils::ComputeFluxesOnBoxFromState (Box const& bx, int ncomp, MFIter& mfi,
                                          Vector<BCRec> const& h_bcrec,
                                          const BCRec* d_bcrec,
                                          int const* iconserv,
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
                                          const EBFArrayBoxFactory& ebfact,
                                          Array4<Real const> const& values_on_eb_inflow,
 #endif
@@ -269,7 +269,7 @@ HydroUtils::ComputeFluxesOnBoxFromState (Box const& bx, int ncomp, MFIter& mfi,
                                 knownFaceState,
                                 AMREX_D_DECL(u_mac, v_mac, w_mac),
                                 divu, fq, geom, l_dt, h_bcrec, d_bcrec, iconserv,
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
                                 ebfact, values_on_eb_inflow,
 #endif
                                 godunov_use_ppm, godunov_use_forces_in_trans,
@@ -297,7 +297,7 @@ HydroUtils::ComputeFluxesOnBoxFromState (Box const& bx, int ncomp, MFIter& mfi,
                                          Vector<BCRec> const& h_bcrec,
                                          const BCRec* d_bcrec,
                                          int const* iconserv,
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
                                          const EBFArrayBoxFactory& ebfact,
                                          Array4<Real const> const& values_on_eb_inflow,
 #endif
@@ -316,7 +316,7 @@ HydroUtils::ComputeFluxesOnBoxFromState (Box const& bx, int ncomp, MFIter& mfi,
                                 AMREX_D_DECL(u_mac, v_mac, w_mac),
                                 AMREX_D_DECL(u_mac, v_mac, w_mac),
                                 divu, fq, geom, l_dt, h_bcrec, d_bcrec, iconserv,
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
                                 ebfact, values_on_eb_inflow,
 #endif
                                 godunov_use_ppm, godunov_use_forces_in_trans,
@@ -347,7 +347,7 @@ HydroUtils::ComputeFluxesOnBoxFromState (Box const& bx, int ncomp, MFIter& mfi,
                                          Vector<BCRec> const& h_bcrec,
                                          const BCRec* d_bcrec,
                                          int const* iconserv,
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
                                          const EBFArrayBoxFactory& ebfact,
                                          Array4<Real const> const& values_on_eb_inflow,
 #endif
@@ -366,7 +366,7 @@ HydroUtils::ComputeFluxesOnBoxFromState (Box const& bx, int ncomp, MFIter& mfi,
                                 AMREX_D_DECL(u_mac, v_mac, w_mac),
                                 AMREX_D_DECL(u_flux, v_flux, w_flux),
                                 divu, fq, geom, l_dt, h_bcrec, d_bcrec, iconserv,
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
                                 ebfact, values_on_eb_inflow,
 #endif
                                 godunov_use_ppm, godunov_use_forces_in_trans,
@@ -397,7 +397,7 @@ HydroUtils::ComputeFluxesOnBoxFromState (Box const& bx, int ncomp, MFIter& mfi,
                                          Vector<BCRec> const& h_bcrec,
                                          const BCRec* d_bcrec,
                                          int const* iconserv,
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
                                          const EBFArrayBoxFactory& ebfact,
                                          Array4<Real const> const& values_on_eb_inflow,
 #endif
@@ -409,7 +409,7 @@ HydroUtils::ComputeFluxesOnBoxFromState (Box const& bx, int ncomp, MFIter& mfi,
                                          amrex::Array4<int const> const& bc_arr)
 
 {
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
     EBCellFlagFab const& flagfab = ebfact.getMultiEBCellFlagFab()[mfi];
     Array4<EBCellFlag const> const& flag = flagfab.const_array();
 
@@ -429,7 +429,7 @@ HydroUtils::ComputeFluxesOnBoxFromState (Box const& bx, int ncomp, MFIter& mfi,
                          divu, fq,
                          geom, l_dt,
                          h_bcrec, d_bcrec, iconserv,
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
                          ebfact, values_on_eb_inflow, regular,
 #endif
                          godunov_use_ppm, godunov_use_forces_in_trans,
@@ -440,7 +440,7 @@ HydroUtils::ComputeFluxesOnBoxFromState (Box const& bx, int ncomp, MFIter& mfi,
     // Compute fluxes.
     // For a typical advection step, the velocity here is the u_mac above.
     // For a multilevel synchronization, the velocity here is the "corrective" velocity.
-#ifdef AMREX_USE_EB
+#if defined(AMREX_USE_EB) && !defined(HYDRO_NO_EB)
     if (!regular) {
         AMREX_D_TERM(Array4<Real const> const& apx = ebfact.getAreaFrac()[0]->const_array(mfi);,
                      Array4<Real const> const& apy = ebfact.getAreaFrac()[1]->const_array(mfi);,