From f7af1be22b54fab59aff1726b18a4ba4c6327f76 Mon Sep 17 00:00:00 2001
From: Raoul Hodgson <raoul.hodgson@ed.ac.uk>
Date: Thu, 26 Oct 2023 14:32:59 +0100
Subject: [PATCH] Added comms and cuda total memory usage to stat db

---
 Hadrons/StatLogger.cpp | 12 ++++++++++--
 Hadrons/StatLogger.hpp |  1 +
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/Hadrons/StatLogger.cpp b/Hadrons/StatLogger.cpp
index e920f41b..ed913649 100644
--- a/Hadrons/StatLogger.cpp
+++ b/Hadrons/StatLogger.cpp
@@ -77,9 +77,11 @@ void StatLogger::setDatabase(Database &db)
             "       deviceMemory.totalCurrent*0.000000953674316 AS totalCurrentMB,                     "
             "       deviceMemory.envCurrent*0.000000953674316 AS envCurrentMB,                         "
             "       deviceMemory.gridCurrent*0.000000953674316 AS gridCurrentMB,                       "
+	    "       deviceMemory.gridCommsCurrent*0.000000953674316 AS gridCommsCurrentMB,             "
             "       deviceMemory.gridCacheCurrent*0.000000953674316 AS gridCacheCurrentMB,             "
             "       deviceMemory.gridTotalCurrent*0.000000953674316 AS gridTotalCurrentMB,             "
             "       deviceMemory.evictableCurrent*0.000000953674316 AS evictableCurrentMB,             "
+	    "       (deviceMemory.totalCurrent-deviceMemory.gridTotalCurrent)*0.000000953674316 AS gridDeficitCurrentMB, "
             "       deviceMemory.hostToDevice*0.000000953674316 AS hostToDeviceMB,                     "
             "       deviceMemory.hostToDevice*0.000000953674316/" + periodSec + " AS hostToDeviceMBps, "
             "       deviceMemory.hostToDeviceTransfers AS hostToDeviceTransfers,                       "
@@ -195,12 +197,18 @@ void StatLogger::logDeviceMemory(const GridTime::rep time)
     Mem               buf;
     DeviceMemoryEntry e;
 
+    size_t free_byte=0;
+    size_t total_byte=0;
+#ifdef GRID_CUDA_NOUVM
+    cudaMemGetInfo( &free_byte, &total_byte ) ;
+#endif
     e.time                  = time;
-    e.totalCurrent          = 0;
+    e.totalCurrent          = total_byte-free_byte;
     e.envCurrent            = 0;
     e.gridCurrent           = MemoryManager::DeviceBytes;
     e.gridCacheCurrent      = MemoryManager::DeviceCacheBytes();
-    e.gridTotalCurrent      = e.gridCurrent + e.gridCacheCurrent;
+    e.gridCommsCurrent      = GlobalSharedMemory::MAX_MPI_SHM_BYTES;
+    e.gridTotalCurrent      = e.gridCurrent + e.gridCacheCurrent + e.gridCommsCurrent;
     e.evictableCurrent      = MemoryManager::DeviceLRUBytes;
     buf.h2d                 = MemoryManager::HostToDeviceBytes;
     buf.h2dTr               = MemoryManager::HostToDeviceXfer;
diff --git a/Hadrons/StatLogger.hpp b/Hadrons/StatLogger.hpp
index ae338405..1c7450b2 100644
--- a/Hadrons/StatLogger.hpp
+++ b/Hadrons/StatLogger.hpp
@@ -59,6 +59,7 @@ class StatLogger
                            SqlNotNull<size_t>, envCurrent,
                            SqlNotNull<size_t>, gridCurrent,
                            SqlNotNull<size_t>, gridCacheCurrent,
+			   SqlNotNull<size_t>, gridCommsCurrent,
                            SqlNotNull<size_t>, gridTotalCurrent,
                            SqlNotNull<size_t>, evictableCurrent,
                            SqlNotNull<size_t>, hostToDevice,