Merge pull request #25 from LLNL/develop

Cfits file format support.

CMakeLists.txt

@@ -1,33 +1,30 @@
 cmake_minimum_required (VERSION 3.5.1)
-project(umap VERSION 0.0.2)
+project(umap 
+  VERSION 0.0.3
+  LANGUAGES CXX C
+)
 
-configure_file(
-  "${PROJECT_SOURCE_DIR}/config/config.h.in"
-  "${PROJECT_BINARY_DIR}/config/config.h"
-  )
-
-if (NOT CMAKE_BUILD_TYPE)
-  message(STATUS "No build type selected, default to Debug")
-  set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Build type (default Debug)" FORCE)
-endif()
+OPTION (ENABLE_CFITS "Build umap with Logging enabled" Off)
+OPTION (ENABLE_FITS_TESTS "Build FITS-based Tests (requires qfits library)" Off)
+OPTION (ENABLE_TESTS "Selects whether tests are built." On)
+OPTION (ENABLE_LOGGING "Build umap with Logging enabled" On)
+OPTION (ENABLE_STATS "Enable display statistics on exit" Off)
 
-OPTION (BUILD_FITS "Build FITS-based Tests (requires qfits library)" FALSE)
+include(cmake/BuildEnv.cmake)
+include(cmake/BuildType.cmake)
+include(cmake/SetupUmapThirdParty.cmake)
 
-set (FLAGS_ALL "-Wall")
-set (FLAGS_DEBUG_ALL "-g -O0 -DDEBUG")
-set (FLAGS_RELEASE_ALL "-O3")
+set(UMAP_ENABLE_LOGGING ${ENABLE_LOGGING})
+set(UMAP_ENABLE_CFITS ${ENABLE_CFITS})
+set(UMAP_DISPLAY_STATS ${ENABLE_STATS})
 
-set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99 ${FLAGS_ALL}")
-set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 ${FLAGS_ALL}")
-set (CMAKE_CXX_FLAGS_DEBUG "${FLAGS_DEBUG_ALL}")
-set (CMAKE_C_FLAGS_DEBUG "${FLAGS_DEBUG_ALL}")
-set (CMAKE_CXX_FLAGS_RELEASE "${FLAGS_RELEASE_ALL}")
-set (CMAKE_C_FLAGS_RELEASE "${FLAGS_RELEASE_ALL}")
+configure_file(
+  ${PROJECT_SOURCE_DIR}/config/config.h.in
+  ${PROJECT_BINARY_DIR}/include/config.h)
 
 add_subdirectory(src)
 
-OPTION (ENABLE_TESTS "Selects whether tests are built." TRUE)
-if (ENABLE_TESTS)
+if ( ENABLE_TESTS )
   add_subdirectory(tests)
 endif()
 

README.md

@@ -1,4 +1,4 @@
-# UMAP v0.0.2 (alpha)
+# UMAP v0.0.3 (alpha)
 
 Umap is a library that provides an mmap()-like interface to a simple, user-
 space page fault handler based on the userfaultfd Linux feature (starting with

cmake/BuildEnv.cmake

@@ -0,0 +1,3 @@
+# Set up the language supported for the build environment
+set (CMAKE_CXX_STANDARD 11)
+set (CMAKE_C_STANDARD 99)

cmake/BuildType.cmake

@@ -0,0 +1,14 @@
+# Set a default build type if none was specified
+set( default_build_type "Release" )
+if ( EXISTS "${CMAKE_SOURCE_DIR}/.git" )
+  set( default_build_type "Debug" )
+endif()
+
+if ( NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES )
+  message(STATUS "Setting build type to '${default_build_type}' as none was specified.")
+  set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE
+      STRING "Choose the type of build." FORCE)
+  # Set the possible values of build type for cmake-gui
+  set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release"
+      "MinSizeRel" "RelWithDebInfo")
+endif()

cmake/SetupUmapThirdParty.cmake

@@ -0,0 +1,21 @@
+if ( ENABLE_CFITS )
+  find_library( CFITS_LIBRARY
+    libcfitsio.a
+    PATHS ${CFITS_LIBRARY_PATH}
+  )
+
+  if ( NOT CFITS_LIBRARY )
+    message( FATAL_ERROR "Could not find CFITS library, make sure CFITS_LIBRARY_PATH is set properly")
+  endif()
+
+  find_path( CFITS_INCLUDE_DIR
+    fitsio.h
+    PATHS ${CFITS_INCLUDE_PATH}
+  )
+
+  if ( NOT CFITS_INCLUDE_DIR )
+    message(FATAL_ERROR "Could not find CFITS include directory, make sure CFITS_INCLUDE_PATH is set properly")
+  endif()
+
+  include_directories( ${CFITS_INCLUDE_DIR} )
+endif()

config/config.h.in

@@ -4,4 +4,5 @@
 #define UMAP_VERSION_MAJOR @umap_VERSION_MAJOR@
 #define UMAP_VERSION_MINOR @umap_VERSION_MINOR@
 #define UMAP_VERSION_PATCH @umap_VERSION_PATCH@
+#cmakedefine UMAP_DISPLAY_STATS
 #endif

include/umap.h

@@ -13,38 +13,60 @@
 #include <sys/types.h>
 #include <sys/mman.h>
 
-typedef struct umap_backing_file {
-    int fd;
-    off_t data_size;
-    off_t data_offset;    /* Offset of data portion in file */
-} umap_backing_file;
-
 #ifdef __cplusplus
 extern "C" {
 #endif
+/** Signatures for application provided callbacks to read/write data from/to 
+ * persistant storage.
+ *
+ * \param region Returned from previous umap() call
+ * \param buf Buffer provided and owned by umap to read data in to
+ * \param nbytes # of bytes to read/write in/from \a buf
+ * \param region_offset Byte offset from beginning of \a region
+ * \returns If successful, the number of bytes read/written in/from \a buf.
+ * Otherwise, -1.
+ *
+ * \b Note: This callback is assumed to be threadsafe.  Since this is
+ * a "C" interface, this will need to be inforced by convention and
+ * implementations that are thread-unsafe may not function correctly.
+ */
+typedef ssize_t (*umap_pstore_read_f_t)(
+    void* region,
+    void* buf,
+    size_t nbytes,
+    off_t region_offset
+    );
 
-/*
- * umap() is a wrapper around mmap(2) and userfaultfd(2) to allow for creating a mapping of pages managed in user-space.
+typedef ssize_t (*umap_pstore_write_f_t)(
+    void* region,
+    void* buf,
+    size_t nbytes,
+    off_t region_offset
+    );
+
+/** Allow application to create region of memory to a peristant store
+ * \param addr Same as input argument for mmap(2)
+ * \param length Same as input argument of mmap(2)
+ * \param prot Same as input argument of mmap(2)
+ * \param flags Same as input argument of mmap(2)
+ * \param r_pstore pointer to callback function to be used for providing data from
+ *                 persistent storage.
+ * \param w_pstore pointer to callback function to be used for saving data to
+ *                 persistent storage.
  */
-void* umap( void*  addr,    /* See mmap(2) */
-            size_t length,  /* See mmap(2) */
-            int    prot,    /* See mmap(2) */
-            int    flags,   /* See below, see mmap(2) for general notes */
-            int    fd,      /* See mmap(2) */
-            off_t  offset   /* See mmap(2) */
-        );
+void* umap(
+    void* addr,
+    size_t length,
+    int prot,
+    int flags,
+    umap_pstore_read_f_t r_pstore,
+    umap_pstore_write_f_t w_pstore
+);
+
 int uunmap( void*  addr,    /* See mmap(2) */
             size_t length   /* See mmap(2) */
         );
 
-void* umap_mf(void* addr, 
-      size_t        length, 
-      int           prot, 
-      int           flags,
-      int           num_backing_files,
-      umap_backing_file* backing_files
-        );
-
 uint64_t umap_cfg_get_bufsize( void );
 void umap_cfg_set_bufsize( uint64_t page_bufsize );
 #ifdef __cplusplus
@@ -61,4 +83,4 @@ void umap_cfg_set_bufsize( uint64_t page_bufsize );
  * Return codes
  */
 #define UMAP_FAILED (void *)-1
-#endif // _UMAP_H_
+#endif // _UMAP_H

src/CMakeLists.txt

@@ -8,7 +8,7 @@ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
 
 include_directories( 
   BEFORE "${CMAKE_CURRENT_SOURCE_DIR}"
-  "${PROJECT_BINARY_DIR}/../config"
+  "${PROJECT_BINARY_DIR}/../include"
   "${CMAKE_CURRENT_SOURCE_DIR}/../include"
   "${CMAKE_CURRENT_SOURCE_DIR}/../sysincludes"
   )

src/umap.cpp

@@ -68,7 +68,7 @@ class UserFaultHandler;
 class _umap {
   friend UserFaultHandler;
   public:
-    _umap(void* _region, uint64_t _rsize, int num_backing_file, umap_backing_file* backing_files);
+    _umap(void* _region, uint64_t _rsize, umap_pstore_read_f_t _ps_read, umap_pstore_write_f_t _ps_write);
     ~_umap();
 
     static inline void* UMAP_PAGE_BEGIN(const void* a) {
@@ -79,9 +79,9 @@ class _umap {
   private:
     void* region;
     uint64_t region_size;
-    int backingfile_fd;
-    vector<umap_backing_file> bk_files;
     bool uffd_time_to_stop_working;
+    umap_pstore_read_f_t pstore_read;
+    umap_pstore_write_f_t pstore_write;
 };
 
 class UserFaultHandler {
@@ -213,36 +213,27 @@ static int check_uffd_compatibility( void )
   return 0;
 }
 
-void* umap(void* addr, uint64_t length, int prot, int flags, int fd, off_t offset)
+void* umap(void* base_addr, uint64_t region_size, int prot, int flags, umap_pstore_read_f_t _ps_read, umap_pstore_write_f_t _ps_write)
 {
-  struct stat file;
-
   if (check_uffd_compatibility() < 0)
     return NULL;
 
-  fstat(fd,&file);
-  struct umap_backing_file file1={.fd = fd, .data_size = file.st_size, .data_offset = offset};
-  return umap_mf(addr, length, prot, flags, 1, &file1);
-}
-
-void* umap_mf(void* bass_addr, uint64_t region_size, int prot, int flags, int num_backing_file, umap_backing_file* backing_files)
-{
-  assert((region_size % page_size) == 0);
+  assert("UMAP: Region size must be multple of page_size" && (region_size % page_size) == 0);
 
   if (!(flags & UMAP_PRIVATE) || flags & ~(UMAP_PRIVATE|UMAP_FIXED)) {
     cerr << "umap: Invalid flags: " << hex << flags << endl;
     return UMAP_FAILED;
   }
 
-  void* region = mmap(bass_addr, region_size, prot, flags | (MAP_ANONYMOUS | MAP_NORESERVE), -1, 0);
+  void* region = mmap(base_addr, region_size, prot, flags | (MAP_ANONYMOUS | MAP_NORESERVE), -1, 0);
 
   if (region == MAP_FAILED) {
     perror("ERROR: mmap failed: ");
     return UMAP_FAILED;
   }
 
   try {
-    active_umaps[region] = new _umap{region, region_size, num_backing_file, backing_files};
+    active_umaps[region] = new _umap{region, region_size, _ps_read, _ps_write};
   } catch(const std::exception& e) {
     cerr << __FUNCTION__ << " Failed to launch _umap: " << e.what() << endl;
     return UMAP_FAILED;
@@ -352,24 +343,28 @@ void __attribute ((destructor)) fine_umap_lib( void )
 //
 // _umap class implementation
 //
-_umap::_umap(void* _region, uint64_t _rsize, int num_backing_file, umap_backing_file* backing_files)
-    : region{_region}, region_size{_rsize}, uffd_time_to_stop_working{false}
+_umap::_umap(void* _region, uint64_t _rsize, umap_pstore_read_f_t _ps_read, umap_pstore_write_f_t _ps_write)
+    : region{_region}, region_size{_rsize}, uffd_time_to_stop_working{false}, pstore_read{_ps_read}, pstore_write{_ps_write}
 {
-  for (int i=0;i<num_backing_file;i++)
-    bk_files.push_back(backing_files[i]); 
-
   uint64_t pages_in_region = region_size / page_size;
   uint64_t pages_per_block = pages_in_region < UMAP_PAGES_PER_BLOCK ? pages_in_region : UMAP_PAGES_PER_BLOCK;
   uint64_t page_blocks = pages_in_region / pages_per_block;
-  uint64_t remainder_of_pages_in_last_block = pages_in_region % pages_per_block;
-
-  if (remainder_of_pages_in_last_block)
-    page_blocks++;          // Account for extra block
 
   uint64_t num_workers = page_blocks < uffd_threads ? page_blocks : uffd_threads;
   uint64_t page_blocks_per_worker = page_blocks / num_workers;
+  uint64_t additional_blocks_for_last_worker = page_blocks % num_workers;
 
-  // cout << "umap " << num_workers << " workers for: " << region << " - " << (void*)((char*)region+region_size) << endl;
+  stringstream ss;
+  ss << "umap(" 
+    << region << " - " << (void*)((char*)region+region_size) 
+    << ") " << pages_in_region << " region pages, "
+    << pages_per_block << " pages per block, "
+    << page_blocks  << " page blocks, "
+    << num_workers << " workers, " 
+    << page_blocks_per_worker << " page blocks per worker, "
+    << additional_blocks_for_last_worker << " additional pages blocks for last worker"
+    << endl;
+  umapdbg("%s\n", ss.str().c_str());
 
   try {
     for (uint64_t worker = 0; worker < num_workers; ++worker) {
@@ -379,8 +374,8 @@ _umap::_umap(void* _region, uint64_t _rsize, int num_backing_file, umap_backing_
       pb.length = page_blocks_per_worker * pages_per_block * page_size;
 
       // If I am the last worker and we have residual pages in last block
-      if ((worker == num_workers-1) && remainder_of_pages_in_last_block)
-        pb.length -= ((pages_per_block - remainder_of_pages_in_last_block)) * page_size;
+      if ((worker == num_workers-1) && additional_blocks_for_last_worker)
+        pb.length += (additional_blocks_for_last_worker * pages_per_block * page_size);
 
       vector<umap_PageBlock> segs{ pb };
 
@@ -461,7 +456,7 @@ UserFaultHandler::UserFaultHandler(_umap* _um, const vector<umap_PageBlock>& _pb
       .mode = UFFDIO_REGISTER_MODE_MISSING | UFFDIO_REGISTER_MODE_WP
     };
 
-    umapdbg("Register %p\n", seg.base);
+    umapdbg("Register %p - %p\n", seg.base, (void*)((uint64_t)seg.base + (uint64_t)(seg.length-1)));
 
     if (ioctl(userfault_fd, UFFDIO_REGISTER, &uffdio_register) == -1) {
       perror("ERROR: ioctl/uffdio_register");
@@ -476,7 +471,6 @@ UserFaultHandler::UserFaultHandler(_umap* _um, const vector<umap_PageBlock>& _pb
     }
   }
 
-  _u->backingfile_fd=_u->bk_files[0].fd;
   uffd_worker = new thread{&UserFaultHandler::uffd_handler, this};
 }
 
@@ -660,14 +654,10 @@ void UserFaultHandler::pagefault_event(const struct uffd_msg& msg)
   //
   // Page not in memory, read it in and (potentially) evict someone
   //
-  int file_id=0;
   off_t offset=(uint64_t)page_begin - (uint64_t)_u->region;
 
-  file_id = offset/_u->bk_files[0].data_size;   //find the file id and offset number
-  offset %= _u->bk_files[0].data_size;
-
-  if (pread(_u->bk_files[file_id].fd, tmppagebuf, page_size, offset+_u->bk_files[file_id].data_offset) == -1) {
-    perror("ERROR: pread failed");
+  if (_u->pstore_read(_u->region, tmppagebuf, page_size, offset) == -1) {
+    perror("ERROR: pstore_read failed");
     exit(1);
   }
 
@@ -736,8 +726,8 @@ void UserFaultHandler::evict_page(umap_page* pb)
     // Prevent further writes.  No need to do this if not dirty because WP is already on.
 
     enable_wp_on_pages_and_wake((uint64_t)page, 1);
-    if (pwrite(_u->backingfile_fd, (void*)page, page_size, (off_t)((uint64_t)page - (uint64_t)_u->region)) == -1) {
-      perror("ERROR: pwrite failed");
+    if (_u->pstore_write(_u->region, (void*)page, page_size, (off_t)((uint64_t)page - (uint64_t)_u->region)) == -1) {
+      perror("ERROR: pstore_write failed");
       assert(0);
     }
   }
@@ -870,6 +860,7 @@ void umap_page::set_page(void* _p)
 //
 void umap_stats::print_stats(void)
 {
+#ifdef UMAP_DISPLAY_STATS
   cerr << stat_faults << " Faults\n"
     << read_faults << " READ Faults" << endl
     << write_faults << " WRITE Faults" << endl
@@ -879,4 +870,5 @@ void umap_stats::print_stats(void)
     << sigbus << " SIGBUS Errors" << endl
     << stuck_wp << " Stuck WP Workarounds" << endl
     << dropped_dups << " Dropped Duplicates" << endl;
+#endif
 }