Merged fix proper nesting changes and fix timers

src/2d/shallow/amr2.f90

@@ -82,8 +82,8 @@ program amr2
     use amr_module, only: tvoll, tvollCPU, rvoll, rvol, mstart, possk, ibuff
     use amr_module, only: timeRegridding,timeUpdating, timeValout
     use amr_module, only: timeBound,timeStepgrid, timeFlagger,timeBufnst,timeFilvalTot
-    use amr_module, only: timeBoundCPU,timeStepGridCPU,timeSetauxCPU,timeRegriddingCPU
-    use amr_module, only: timeSetaux, timeSetauxCPU, timeValoutCPU
+    use amr_module, only: timeBoundCPU,timeStepGridCPU,timeRegriddingCPU
+    use amr_module, only: timeValoutCPU,timeTick,timeTickCPU
     use amr_module, only: kcheck, iorder, lendim, lenmax
 
     use amr_module, only: dprint, eprint, edebug, gprint, nprint, pprint
@@ -116,7 +116,7 @@ program amr2
 
     ! Timing variables
     integer :: clock_start, clock_finish, clock_rate, ttotal
-    real(kind=8) :: cpu_start, cpu_finish, ttotalcpu
+    real(kind=8) :: ttotalcpu
 
     ! Common block variables
     real(kind=8) :: dxmin, dymin
@@ -594,8 +594,9 @@ program amr2
     call conck(1,nvar,naux,time,rest)
 
     ! Timing
+    ! moved inside tick, so timers can be checkpoint for
+    ! possible restart
     call system_clock(clock_start,clock_rate)
-    call cpu_time(cpu_start)
 
     if (output_t0) then
         call valout(1,lfine,time,nvar,naux)
@@ -618,7 +619,6 @@ program amr2
 
 
     call system_clock(clock_finish,clock_rate)
-    call cpu_time(cpu_finish)
 
     !output timing data
     write(*,*)
@@ -701,11 +701,11 @@ program amr2
     !Total Time
     format_string="('Total time:   ',1f15.3,'        ',1f15.3,'  ')"
     write(outunit,format_string) &
-            real(clock_finish - clock_start,kind=8) / real(clock_rate,kind=8), &
-            cpu_finish-cpu_start
+            real(timeTick,kind=8) / real(clock_rate,kind=8), &
+            timeTickCPU         
     write(*,format_string) &
-            real(clock_finish - clock_start,kind=8) / real(clock_rate,kind=8), &
-            cpu_finish-cpu_start
+            real(timeTick,kind=8) / real(clock_rate,kind=8), &
+            timeTickCPU         
 
     format_string="('Using',i3,' thread(s)')"
     write(outunit,format_string) maxthreads
@@ -720,6 +720,10 @@ program amr2
     write(outunit,"('Note: The CPU times are summed over all threads.')")
     write(*,"('      Total time includes more than the subroutines listed above')")
     write(outunit,"('      Total time includes more than the subroutines listed above')")
+    if (rest) then
+      write(*,"('      Times for restart runs are cumulative')")
+      write(outunit,"('      Times for restart runs are cumulative')")
+    endif
 
 
     !end of timing data

src/2d/shallow/check.f

@@ -74,7 +74,13 @@ subroutine check(nsteps,time,nvar,naux)
      3          numgrids,kcheck,nsteps,
      3          time,matlabu
       write(chkunit) avenumgrids, iregridcount,
-     1               evol,rvol,rvoll,lentot,tmass0,cflmax
+     1               evol,rvol,rvoll,lentot,tmass0,cflmax,
+     2               tvoll,tvollCPU,timeTick,timeTickCPU,
+     3               timeStepgrid,timeStepgridCPU,
+     4               timeBound,timeBoundCPU,
+     5               timeRegridding,timeRegriddingCPU,
+     6               timeValout,timeValoutCPU
+
 c
 c     ### new capability to dump the fgmax data, if exists
 c     ### fortran requires specifying each component, if

src/2d/shallow/filpatch.f90

@@ -17,7 +17,7 @@ recursive subroutine filrecur(level,nvar,valbig,aux,naux,t,mx,my, &
     use amr_module, only: nghost, xlower, ylower, xupper, yupper, outunit
     use amr_module, only: xperdom, yperdom, spheredom, hxposs, hyposs
     use amr_module, only: intratx, intraty, iregsz, jregsz
-    use amr_module, only: timeSetaux, NEEDS_TO_BE_SET
+    use amr_module, only: NEEDS_TO_BE_SET
 
     use geoclaw_module, only: sea_level, dry_tolerance
     use topo_module, only: topo_finalized
@@ -35,7 +35,6 @@ recursive subroutine filrecur(level,nvar,valbig,aux,naux,t,mx,my, &
     real(kind=8), intent(in out) :: aux(naux,mx,my)
 
     ! Local storage
-    integer  :: clock_start, clock_finish, clock_rate, clock_dif
     integer  :: iplo, iphi, jplo, jphi
 
     ! Flagging of set cells
@@ -179,7 +178,6 @@ recursive subroutine filrecur(level,nvar,valbig,aux,naux,t,mx,my, &
 
         if (naux > 0) then
             nghost_patch = 0
-            call system_clock(clock_start,clock_rate)
 
             ! update topography if needed
             !if ((num_dtopo>0).and.(topo_finalized.eqv..false.)) then
@@ -196,10 +194,6 @@ recursive subroutine filrecur(level,nvar,valbig,aux,naux,t,mx,my, &
             call setaux(nghost_patch, mx_coarse, my_coarse,       &
                         xlow_coarse, ylow_coarse,                 &
                         dx_coarse,dy_coarse,naux,auxcrse)
-            call system_clock(clock_finish,clock_rate)
-            clock_dif = clock_finish - clock_start
-!$OMP ATOMIC            
-            timeSetaux = timeSetaux + clock_dif
         endif
 
         ! Fill in the edges of the coarse grid

src/2d/shallow/filval.f90

@@ -12,11 +12,12 @@
 !
 subroutine filval(val, mitot, mjtot, dx, dy, level, time,  mic, &
                   mjc, xleft, xright, ybot, ytop, nvar, mptr, ilo, ihi, &
-                  jlo, jhi, aux, naux,  sp_over_h, thisSetauxTime)
+                  jlo, jhi, aux, naux,  sp_over_h )
 
     use amr_module, only: xlower, ylower, intratx, intraty, nghost, xperdom
     use amr_module, only: yperdom, spheredom, xupper, yupper, alloc
     use amr_module, only: outunit, NEEDS_TO_BE_SET
+    use amr_module, only: newstl, iregsz, jregsz
 
     use topo_module, only: aux_finalized
     use geoclaw_module, only: dry_tolerance, sea_level
@@ -33,7 +34,6 @@ subroutine filval(val, mitot, mjtot, dx, dy, level, time,  mic, &
     ! Output
     real(kind=8), intent(in out) :: sp_over_h
     real(kind=8), intent(in out) :: val(nvar,mitot,mjtot), aux(naux,mitot,mjtot)
-    integer, intent (out) ::  thisSetauxTime
 
     ! Local storage
     integer :: refinement_ratio_x, refinement_ratio_y, iclo, jclo, ichi, jchi, ng, i, ico, ifine
@@ -45,13 +45,19 @@ subroutine filval(val, mitot, mjtot, dx, dy, level, time,  mic, &
     logical :: fineflag(3)
     real(kind=8) :: fliparray((mitot+mjtot)*(nvar+naux))
     real(kind=8) :: aux2(naux,mitot,mjtot)
-    integer :: clock_start, clock_finish, clock_rate
     integer :: nx, ny
     real(kind=8) setflags(mitot,mjtot),maxauxdif
+    integer :: jm, im, nm
+    logical :: sticksoutxfine, sticksoutyfine,sticksoutxcrse,sticksoutycrse
+    logical :: DIAGONAL_CORNER
 
     ! External function definitions
     real(kind=8) :: get_max_speed
 
+    DIAGONAL_CORNER(im,jm,mic,mjc) = (im .eq. 1   .and. jm .eq. mjc)  .or.     &
+                                      (im .eq. mic .and. jm .eq. mjc) .or.     &
+                                      (im .eq. 1   .and. jm .eq. 1)    .or.     &
+                                      (im .eq. mic .and. jm .eq. 1)
 
     refinement_ratio_x = intratx(level-1)
     refinement_ratio_y = intraty(level-1)
@@ -74,16 +80,23 @@ subroutine filval(val, mitot, mjtot, dx, dy, level, time,  mic, &
     jchi   = (jhi + 1) / refinement_ratio_y - 1 + 1
     ng     = 0
 
+
+    sticksoutxfine = ( (ilo .lt. 0) .or. (ihi .ge. iregsz(level)))
+    sticksoutyfine = ( (jlo .lt. 0) .or. (jhi .ge. jregsz(level)))
+    sticksoutxcrse = ((iclo .lt. 0) .or. (ichi .ge. iregsz(level-1)))
+    sticksoutycrse = ((jclo .lt. 0) .or. (jchi .ge. jregsz(level-1)))
+
+
     if (naux == 0) then
         write(*,*)" in filval/geoclaw with naux=0:  how could this happen?"
-        if (xperdom .or. yperdom .or. spheredom) then
+        if ((xperdom .and. sticksoutxcrse) .or. (yperdom.and. sticksoutycrse) .or. spheredom) then
             call preintcopy(valc,mic,mjc,nvar,iclo,ichi,jclo,jchi,level-1,fliparray)
         else
             call intcopy(valc,mic,mjc,nvar,iclo,ichi,jclo,jchi,level-1,1,1)
         endif
     else  
         ! intersect grids and copy all (soln and aux)
-        if (xperdom .or. yperdom .or. spheredom) then
+        if ((xperdom .and. sticksoutxcrse) .or. (yperdom.and. sticksoutycrse) .or. spheredom) then
             call preicall(valc,auxc,mic,mjc,nvar,naux,iclo,ichi,jclo,jchi, &
                           level-1,fliparray)
         else
@@ -119,7 +132,7 @@ subroutine filval(val, mitot, mjtot, dx, dy, level, time,  mic, &
 
        if (naux .gt. 0) then 
              aux(1,:,:) = NEEDS_TO_BE_SET  ! will indicate fine cells not yet set
-             if (xperdom .or. yperdom) then
+             if ((xperdom .and. sticksoutxfine)  .or. (yperdom.and.sticksoutyfine)) then
                 call preicall(val,aux,mitot,mjtot,nvar,naux,ilo-nghost,ihi+nghost,  &
                               jlo-nghost,jhi+nghost,level,fliparray)  
              else
@@ -130,14 +143,11 @@ subroutine filval(val, mitot, mjtot, dx, dy, level, time,  mic, &
              ! need this so we know where to use coarse grid to set fine solution w/o overwriting
              if (aux_finalized .lt. 2) aux(1,:,:) = NEEDS_TO_BE_SET  ! reset entire aux array since topo moving
                !set remaining aux vals not set by copying from prev existing grids
-               call system_clock(clock_start,clock_rate)
                call setaux(nghost,nx,ny,xleft,ybot,dx,dy,naux,aux)
-               call system_clock(clock_finish,clock_rate)
-               thisSetauxTime = thisSetauxTime + clock_finish - clock_start
        else ! either no aux exists, or cant reuse yet  
           ! if topo not final, then setaux called in gfixup before this routine
           ! so only call intcopy (which copies soln) and not icall.
-          if (xperdom .or. yperdom) then
+          if ((xperdom .and. sticksoutxfine)  .or. (yperdom.and.sticksoutyfine)) then
              call preintcopy(val,mitot,mjtot,nvar,ilo-nghost,ihi+nghost,  &
                        jlo-nghost,jhi+nghost,level,1,1,fliparray)
           else

src/2d/shallow/gfixup.f

@@ -99,14 +99,13 @@ subroutine gfixup(lbase, lfnew, nvar, naux, newnumgrids,
 !$OMP&            PRIVATE(clock_start,clock_finish,clock_rate)
 !$OMP&            PRIVATE(j,mptr,nx,ny,mitot,mjtot,corn1,corn2,loc)
 !$OMP&            PRIVATE(locaux,time,mic,mjc,xl,xr,yb,yt,ilo,ihi)
-!$OMP&            PRIVATE(jlo,jhi,sp_over_h,thisSetauxTime)
+!$OMP&            PRIVATE(jlo,jhi,sp_over_h)
 !$OMP&            SHARED(newnumgrids,listnewgrids,nghost,node,hx,hy)
 !$OMP&            SHARED(rnode,intratx,intraty,lcheck,nvar,alloc,naux)
 !$OMP&            REDUCTION(MAX:this_spoh)
 !$OMP&            REDUCTION(+:timeFilval)
 !$OMP&            SCHEDULE(dynamic,1)
 !$OMP&            DEFAULT(none)
-!$OMP&            REDUCTION(+:timeSetaux)
 
       do  j = 1, newnumgrids(lcheck)
           mptr = listnewgrids(j)
@@ -159,7 +158,7 @@ subroutine gfixup(lbase, lfnew, nvar, naux, newnumgrids,
      2                 xl,xr,yb,yt,nvar,
      3                 mptr,ilo,ihi,jlo,jhi,
      4                 alloc(locaux),naux,
-     5                 sp_over_h,thisSetauxTime)
+     5                 sp_over_h)
            call system_clock(clock_finish,clock_rate)
            timeFilval = timeFilval + clock_finish - clock_start
            this_spoh = max(this_spoh, sp_over_h)

src/2d/shallow/ginit.f

@@ -7,10 +7,6 @@ subroutine ginit(msave, first, nvar, naux, start_time)
       implicit double precision (a-h,o-z)
       logical first
 
-      !for setaux timing
-      integer :: clock_start, clock_finish, clock_rate
-      real(kind=8) :: cpu_start, cpu_finish
-
 
 c ::::::::::::::::::::::::::::: GINIT ::::::::::::::::::::::::
 c
@@ -42,14 +38,8 @@ subroutine ginit(msave, first, nvar, naux, start_time)
                    alloc(locaux+k-1) = NEEDS_TO_BE_SET ! new system checks this val before setting
                 end do
 
-                call system_clock(clock_start,clock_rate)
-                call cpu_time(cpu_start)
                 call setaux(nghost,nx,ny,corn1,corn2,hx,hy,
      &                    naux,alloc(locaux))
-                call system_clock(clock_finish,clock_rate)
-                call cpu_time(cpu_finish)
-                timeSetaux = timeSetaux + clock_finish - clock_start
-                timeSetauxCPU = timeSetauxCPU + cpu_finish - cpu_start
               else 
                 locaux = 1
               endif

src/2d/shallow/restrt.f

@@ -56,7 +56,13 @@ subroutine restrt(nsteps,time,nvar,naux)
      3       numgrids,kcheck1,nsteps,time,
      3       matlabu
       read(rstunit) avenumgrids, iregridcount,
-     1              evol,rvol,rvoll,lentot,tmass0,cflmax
+     1              evol,rvol,rvoll,lentot,tmass0,cflmax,
+     2              tvoll,tvollCPU,timeTick,timeTickCPU,
+     3              timeStepgrid,timeStepgridCPU,
+     4              timeBound,timeBoundCPU,
+     5              timeRegridding,timeRegriddingCPU,
+     6              timeValout,timeValoutCPU
+
 
 c     ### if fgmax grids exist, read them back in.
 c     ### ASSUMES fgmax stuff already set up (main calls set_fgmax) 

src/2d/shallow/tick.f

@@ -20,6 +20,7 @@ subroutine tick(nvar,cut,nstart,vtime,time,naux,start_time,
       logical vtime,dumpout/.false./,dumpchk/.false./,rest,dump_final
       dimension dtnew(maxlv), ntogo(maxlv), tlevel(maxlv)
       integer clock_start, clock_finish, clock_rate
+      integer tick_clock_start, tick_clock_finish, tick_clock_rate
       character(len=128) :: time_format
       real(kind=8) cpu_start,cpu_finish
 
@@ -50,6 +51,9 @@ subroutine tick(nvar,cut,nstart,vtime,time,naux,start_time,
 c          have its error estimated and finer levels should be regridded.
 c ::::::::::::::::::::::::::::::::::::;::::::::::::::::::::::::::
 c
+      call system_clock(tick_clock_start,tick_clock_rate)
+      call cpu_time(tick_cpu_start)
+
 
       ncycle         = nstart
       call setbestsrc()     ! need at very start of run, including restart
@@ -452,7 +456,14 @@ subroutine tick(nvar,cut,nstart,vtime,time,naux,start_time,
 c  # checkpoint everything for possible future restart
 c  # (unless we just did it based on dumpchk)
 c
+      call system_clock(tick_clock_finish,tick_clock_rate)
+      call cpu_time(tick_cpu_finish)
+      timeTick = timeTick + tick_clock_finish - tick_clock_start 
+      timeTickCPU = timeTickCPU + tick_cpu_finish - tick_cpu_start 
+
 
+c  # checkpoint everything for possible future restart
+c  # (unless we just did it based on dumpchk)
 c
       if (checkpt_style .ne. 0) then  ! want a chckpt
          ! check if just did it so dont do it twice