add timeout for data transfer loop, Nx mounting bis, hdd, save_system…

…, save_cache.

the data transfer loop now has a timeout of 1ms. this was added based on the theory that
if the disk speed was slower than the network, then the data loop would block until
the entire file / directory is sent.
usually this wouldnt be the case, until i was transfering files from a hdd.
the seek time can be pretty bad, and thus the network is faster.

many new nx mount options were added, the biggest one being hdd. the impl isn't perfect yet, and it
can block the main thread for quite a while as the disk starts to spin up for the first time.
however i decided that blocking is the best as otherwise opening a folder would return an
error as it isn;t ready yet, however to the user, there is no clear reason as to why it didnt open.
by blocking, the user will likely hear the disk spinning up and flashing, and understand why its
taking some time.

CMakeLists.txt

@@ -184,9 +184,64 @@ ftp_set_compile_definitions(ftpsrv)
 
 if (NINTENDO_SWITCH)
     ftp_set_options(ftpsrv 769 128 1024*64)
-    # ftp_set_options(ftpsrv 769 5 1024*16)
     fetch_minini()
 
+    set(USBHSFS_GPL ON)
+    FetchContent_Declare(libusbhsfs
+        GIT_REPOSITORY https://github.com/DarkMatterCore/libusbhsfs.git
+        GIT_TAG v0.2.9
+    )
+
+    FetchContent_MakeAvailable(libusbhsfs)
+    add_library(libusbhsfs
+        ${libusbhsfs_SOURCE_DIR}/source/usbhsfs_drive.c
+        ${libusbhsfs_SOURCE_DIR}/source/usbhsfs_log.c
+        ${libusbhsfs_SOURCE_DIR}/source/usbhsfs_manager.c
+        ${libusbhsfs_SOURCE_DIR}/source/usbhsfs_mount.c
+        ${libusbhsfs_SOURCE_DIR}/source/usbhsfs_request.c
+        ${libusbhsfs_SOURCE_DIR}/source/usbhsfs_scsi.c
+        ${libusbhsfs_SOURCE_DIR}/source/usbhsfs_utils.c
+    )
+
+    target_include_directories(libusbhsfs PUBLIC ${libusbhsfs_SOURCE_DIR}/include)
+
+    # fatfs stuff
+    target_sources(libusbhsfs PRIVATE
+        ${libusbhsfs_SOURCE_DIR}/source/fatfs/diskio.c
+        ${libusbhsfs_SOURCE_DIR}/source/fatfs/ff_dev.c
+        ${libusbhsfs_SOURCE_DIR}/source/fatfs/ff.c
+        ${libusbhsfs_SOURCE_DIR}/source/fatfs/ffsystem.c
+        ${libusbhsfs_SOURCE_DIR}/source/fatfs/ffunicode.c
+    )
+
+    # sxos stuff
+    target_sources(libusbhsfs PRIVATE
+        ${libusbhsfs_SOURCE_DIR}/source/sxos/usbfs_dev.c
+        ${libusbhsfs_SOURCE_DIR}/source/sxos/usbfs.c
+    )
+
+    if (USBHSFS_GPL)
+        target_sources(libusbhsfs PRIVATE
+            ${libusbhsfs_SOURCE_DIR}/source/lwext4/ext_dev.c
+            ${libusbhsfs_SOURCE_DIR}/source/lwext4/ext_disk_io.c
+            ${libusbhsfs_SOURCE_DIR}/source/lwext4/ext.c
+
+            ${libusbhsfs_SOURCE_DIR}/source/ntfs-3g/ntfs_dev.c
+            ${libusbhsfs_SOURCE_DIR}/source/ntfs-3g/ntfs_disk_io.c
+            ${libusbhsfs_SOURCE_DIR}/source/ntfs-3g/ntfs.c
+        )
+
+        find_library(ntfs-3g_lib ntfs-3g REQUIRED)
+        find_path(ntfs-3g_inc ntfs-3g REQUIRED)
+
+        find_library(lwext4_lib lwext4 REQUIRED)
+        find_path(lwext4_inc lwext4 REQUIRED)
+
+        target_link_libraries(libusbhsfs PRIVATE ${ntfs-3g_lib} ${lwext4_lib})
+        target_include_directories(libusbhsfs PRIVATE ${ntfs-3g_inc} ${lwext4_inc} ${lwext4_inc}/lwext4)
+        target_compile_definitions(libusbhsfs PRIVATE GPL_BUILD)
+    endif()
+
     target_compile_definitions(ftpsrv PUBLIC
         FTP_VFS_HEADER="${CMAKE_CURRENT_SOURCE_DIR}/src/platform/nx/vfs_nx.h"
         FTP_SOCKET_HEADER="${CMAKE_CURRENT_SOURCE_DIR}/src/platform/unistd/socket_unistd.h"
@@ -197,18 +252,19 @@ if (NINTENDO_SWITCH)
         src/platform/nx/vfs_nx.c
         src/platform/nx/vfs/vfs_nx_root.c
         src/platform/nx/vfs/vfs_nx_fs.c
-        src/platform/nx/vfs/vfs_nx_ncm.c
         src/platform/nx/vfs/vfs_nx_save.c
         src/platform/nx/utils.c
         src/log/log.c
     )
 
     add_executable(ftpexe
         src/platform/nx/main.c
+        src/platform/nx/vfs/vfs_nx_hdd.c
         ${NX_SRC}
     )
+    target_compile_definitions(ftpexe PUBLIC USE_USBHSFS=1)
     ftp_add(ftpexe)
-    target_link_libraries(ftpexe PRIVATE ftpsrv minIni)
+    target_link_libraries(ftpexe PRIVATE ftpsrv minIni libusbhsfs)
 
     nx_generate_nacp(
         OUTPUT ftpexe.nacp

assets/config.ini.template

@@ -32,11 +32,15 @@ log = 0
 [Nx]
 # enables showing all available mount points at root "/"
 # for example, SdCard is as at "/sdmc:"
-mount_devices = 1
+mount_devices = 0
 
 # allows save data to be writable, needs mount_devices = 1
 save_writable = 0
 
+# allows for bis partitions to be mounted, needs mount_devices = 1
+# WARNING: modifying bis files can soft brick your switch!
+mount_bis = 0
+
 # uncomment to override the port for application
 # app_port = 21
 

src/ftpsrv.c

@@ -17,6 +17,7 @@
 #include <unistd.h>
 #include <fcntl.h>
 #include <sys/stat.h>
+#include <sys/time.h>
 #include <errno.h>
 
 #if defined(HAVE_IPTOS_THROUGHPUT) && HAVE_IPTOS_THROUGHPUT
@@ -180,6 +181,12 @@ static int strncasecmp(const char* a, const char* b, size_t len) {
 }
 #endif
 
+static size_t ftp_get_timestamp_ms(void) {
+    struct timeval ts;
+    gettimeofday(&ts, NULL);
+    return (ts.tv_sec * 1000000UL + ts.tv_usec) / 1000UL;
+}
+
 static void ftp_log_callback(enum FTP_API_LOG_TYPE type, const char* msg) {
     if (g_ftp.cfg.log_callback) {
         g_ftp.cfg.log_callback(type, msg);
@@ -640,6 +647,7 @@ static enum FTP_FILE_TRANSFER_STATE ftp_file_data_transfer_progress(struct FtpSe
 static void ftp_data_transfer_progress(struct FtpSession* session) {
     struct FtpTransfer* transfer = &session->transfer;
     enum FTP_FILE_TRANSFER_STATE state = FTP_FILE_TRANSFER_STATE_CONTINUE;
+    const size_t start = ftp_get_timestamp_ms();
 
     while (state == FTP_FILE_TRANSFER_STATE_CONTINUE) {
         if (transfer->mode == FTP_TRANSFER_MODE_RETR || transfer->mode == FTP_TRANSFER_MODE_STOR) {
@@ -651,6 +659,11 @@ static void ftp_data_transfer_progress(struct FtpSession* session) {
         if (g_ftp.cfg.progress_callback) {
             g_ftp.cfg.progress_callback();
         }
+
+        // break out if 1ms has elapsed as to not block for too long.
+        if (ftp_get_timestamp_ms() - start >= 1) {
+            break;
+        }
     }
 
     if (state == FTP_FILE_TRANSFER_STATE_ERROR) {

src/log/log.c

@@ -1,5 +1,7 @@
 #include "log.h"
 #include "ftpsrv_vfs.h"
+#include <stdio.h>
+#include <stdarg.h>
 #include <string.h>
 
 static struct FtpVfsFile g_log_file = {0};
@@ -17,6 +19,17 @@ void log_file_write(const char* msg) {
     }
 }
 
+void log_file_fwrite(const char* fmt, ...) {
+    if (g_has_log_file) {
+        char buf[256];
+        va_list va;
+        va_start(va, fmt);
+        vsnprintf(buf, sizeof(buf), fmt, va);
+        va_end(va);
+        log_file_write(buf);
+    }
+}
+
 void log_file_init(const char* path, const char* init_msg) {
     if (!g_has_log_file) {
         ftp_vfs_unlink(path);

src/log/log.h

@@ -10,6 +10,7 @@ extern "C" {
 #endif
 
 void log_file_write(const char* msg);
+void log_file_fwrite(const char* fmt, ...);
 void log_file_init(const char* path, const char* init_msg);
 void log_file_exit(void);
 

src/platform/nx/main.c

@@ -145,6 +145,7 @@ int main(int argc, char** argv) {
     g_ftpsrv_config.timeout = ini_getl("Network", "timeout", 0, INI_PATH);
     const bool log_enabled = ini_getbool("Log", "log", 0, INI_PATH);
     const bool mount_devices = ini_getbool("Nx", "mount_devices", 1, INI_PATH);
+    const bool mount_bis = ini_getbool("Nx", "mount_bis", 0, INI_PATH);
     const bool save_writable = ini_getbool("Nx", "save_writable", 0, INI_PATH);
     g_led_enabled = ini_getbool("Nx", "led", 1, INI_PATH);
     g_ftpsrv_config.port = ini_getl("Nx", "app_port", g_ftpsrv_config.port, INI_PATH);
@@ -153,19 +154,6 @@ int main(int argc, char** argv) {
         log_file_init(LOG_PATH, "ftpsrv - " FTPSRV_VERSION_HASH " - NX-app");
     }
 
-    vfs_nx_init(mount_devices, save_writable);
-    if (mount_devices) {
-        fsdev_wrapMountImage("image_nand", FsImageDirectoryId_Nand);
-        fsdev_wrapMountImage("image_sd", FsImageDirectoryId_Sd);
-
-        // add some shortcuts.
-        FsFileSystem* sdmc = fsdev_wrapGetDeviceFileSystem("sdmc");
-        if (sdmc) {
-            fsdev_wrapMountDevice("switch", "/switch", *sdmc, false);
-            fsdev_wrapMountDevice("contents", "/atmosphere/contents", *sdmc, false);
-        }
-    }
-
     if (!user_len && !pass_len && !g_ftpsrv_config.anon) {
         return error_loop("User / Pass / Anon not set in config!");
     }
@@ -179,6 +167,8 @@ int main(int argc, char** argv) {
         return error_loop("failed to get current ip address");
     }
 
+    vfs_nx_init(mount_devices, save_writable, mount_bis);
+
     const struct in_addr addr = {ip};
     printf(TEXT_YELLOW "ip: %s\n", inet_ntoa(addr));
     printf(TEXT_YELLOW "port: %d" TEXT_NORMAL "\n", g_ftpsrv_config.port);
@@ -229,8 +219,6 @@ int main(int argc, char** argv) {
     }
 }
 
-u32 __nx_fs_num_sessions = 2;
-
 #define TCP_TX_BUF_SIZE (1024 * 64)
 #define TCP_RX_BUF_SIZE (1024 * 64)
 #define TCP_TX_BUF_SIZE_MAX (1024 * 1024 * 4)
@@ -320,10 +308,11 @@ void userAppInit(void) {
         diagAbortWithResult(rc);
     if (R_FAILED(rc = ncmInitialize()))
         diagAbortWithResult(rc);
+    if (R_FAILED(rc = setInitialize()))
+        diagAbortWithResult(rc);
     if (R_FAILED(rc = fsdev_wrapMountSdmc()))
         diagAbortWithResult(rc);
 
-
     // the below doesnt matter if they fail to init.
     hidsysInitialize();
     appletSetAutoSleepDisabled(true);
@@ -334,7 +323,9 @@ void userAppExit(void) {
     log_file_exit();
     vfs_nx_exit();
     consoleExit(NULL);
+
     hidsysExit();
+    setExit();
     ncmExit();
     accountExit();
     socketExit();

src/platform/nx/main_sysmod.c

@@ -36,6 +36,7 @@ int main(void) {
     g_ftpsrv_config.timeout = ini_getl("Network", "timeout", 0, INI_PATH);
     const bool log_enabled = ini_getbool("Log", "log", 0, INI_PATH);
     const bool mount_devices = ini_getbool("Nx", "mount_devices", 1, INI_PATH);
+    const bool mount_bis = ini_getbool("Nx", "mount_bis", 0, INI_PATH);
     const bool save_writable = ini_getbool("Nx", "save_writable", 0, INI_PATH);
     g_led_enabled = ini_getbool("Nx", "led", 1, INI_PATH);
     g_ftpsrv_config.port = ini_getl("Nx", "sys_port", g_ftpsrv_config.port, INI_PATH);
@@ -44,25 +45,14 @@ int main(void) {
         log_file_init(LOG_PATH, "ftpsrv - " FTPSRV_VERSION_HASH " - NX-sys");
     }
 
-    vfs_nx_init(mount_devices, save_writable);
-    if (mount_devices) {
-        fsdev_wrapMountImage("image_nand", FsImageDirectoryId_Nand);
-        fsdev_wrapMountImage("image_sd", FsImageDirectoryId_Sd);
-
-        // add some shortcuts.
-        FsFileSystem* sdmc = fsdev_wrapGetDeviceFileSystem("sdmc");
-        if (sdmc) {
-            fsdev_wrapMountDevice("switch", "/switch", *sdmc, false);
-            fsdev_wrapMountDevice("contents", "/atmosphere/contents", *sdmc, false);
-        }
-    }
-
     // exit early as this is a security risk due to ldn-mitm.
     if (!user_len && !pass_len && !g_ftpsrv_config.anon) {
         log_file_write("User / Pass / Anon not set in config!");
         return EXIT_FAILURE;
     }
 
+    vfs_nx_init(mount_devices, save_writable, mount_bis);
+
     int timeout = -1;
     if (g_ftpsrv_config.timeout) {
         timeout = 1000 * g_ftpsrv_config.timeout;
@@ -199,6 +189,8 @@ void __appInit(void) {
         diagAbortWithResult(rc);
     if (R_FAILED(rc = ncmInitialize()))
         diagAbortWithResult(rc);
+    if (R_FAILED(rc = setInitialize()))
+        diagAbortWithResult(rc);
 
     hidsysInitialize();
     __libnx_init_time();
@@ -210,6 +202,7 @@ void __appExit(void) {
     vfs_nx_exit();
     log_file_exit();
     hidsysExit();
+    setExit();
     ncmExit();
     accountExit();
     socketExit();

src/platform/nx/utils.c

@@ -173,7 +173,7 @@ int fsdev_wrapMountDevice(const char *name, const char* shortcut, FsFileSystem f
             g_fsdev_entries[i].fs = fs;
             g_fsdev_entries[i].shortcut = shortcut;
             g_fsdev_entries[i].own = own;
-            strncpy(g_fsdev_entries[i].path, name, sizeof(g_fsdev_entries[i].path));
+            snprintf(g_fsdev_entries[i].path, sizeof(g_fsdev_entries[i].path), name);
             return 0;
         }
     }