Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use memory mapped files access #1781

Draft
wants to merge 7 commits into
base: master
Choose a base branch
from
Draft

Use memory mapped files access #1781

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e687a27
Test
lhog Nov 25, 2024
0b737db
more linking
p2004a Nov 26, 2024
59266c4
Replace std::ifstream with mio::ummap_source
lhog Nov 26, 2024
044c9f8
Migrate the rest of std::ifstream to mio
lhog Nov 26, 2024
8766d1f
Actually add the library
lhog Nov 26, 2024
5a59485
Fixes
lhog Nov 27, 2024
9a22f3e
Possibly dealt with the daylight saving archives rescan
lhog Nov 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions rts/System/FileSystem/ArchiveScanner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -604,11 +604,11 @@ std::string CArchiveScanner::SearchMapFile(const IArchive* ar, std::string& erro

// check for smf and if the uncompression of important files is too costy
for (unsigned fid = 0; fid != ar->NumFiles(); ++fid) {
const std::pair<std::string, int>& info = ar->FileInfo(fid);
const std::string& ext = FileSystem::GetExtension(StringToLower(info.first));
const auto& [name, size] = ar->FileInfo(fid);
const auto ext = FileSystem::GetExtension(StringToLower(name));

if (ext == "smf")
return info.first;
return name;
}

return "";
Expand Down Expand Up @@ -870,9 +870,9 @@ bool CArchiveScanner::CheckCachedData(const std::string& fullName, unsigned& mod

bool CArchiveScanner::ScanArchiveLua(IArchive* ar, const std::string& fileName, ArchiveInfo& ai, std::string& err)
{
std::vector<std::uint8_t> buf;

if (!ar->GetFile(fileName, buf) || buf.empty()) {
// use the same buffer to avoid allocation costs for every invocation
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Perhaps this could be static to the function instead of class member?

scanArchiveBuffer.clear();
if (!ar->GetFile(fileName, scanArchiveBuffer) || scanArchiveBuffer.empty()) {
err = "Error reading " + fileName;

if (ar->GetArchiveFile().find(".sdp") != std::string::npos)
Expand All @@ -882,7 +882,7 @@ bool CArchiveScanner::ScanArchiveLua(IArchive* ar, const std::string& fileName,
}

// NB: skips LuaConstGame::PushEntries(L) since that would invoke ScanArchive again
LuaParser p(std::string((char*)(buf.data()), buf.size()), SPRING_VFS_ZIP);
LuaParser p(std::string((char*)(scanArchiveBuffer.data()), scanArchiveBuffer.size()), SPRING_VFS_ZIP);

if (!p.Execute()) {
err = "Error in " + fileName + ": " + p.GetErrorLog();
Expand Down
2 changes: 2 additions & 0 deletions rts/System/FileSystem/ArchiveScanner.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,8 @@ class CArchiveScanner
static bool CheckCompression(const IArchive* ar, const std::string& fullName, std::string& error);

private:
std::vector<std::uint8_t> scanArchiveBuffer;

spring::unordered_map<std::string, size_t> archiveInfosIndex;
spring::unordered_map<std::string, size_t> brokenArchivesIndex;

Expand Down
6 changes: 2 additions & 4 deletions rts/System/FileSystem/FileHandler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,9 +133,8 @@ bool CFileHandler::TryReadFromRawFS(const string& fileName)
}

return true;
#else
return false;
#endif
return false;
Copy link
Collaborator

@sprunk sprunk Nov 28, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The non-TOOLS path will have two unconditional returns in a row now, the second one is dead code and may confuse static analysis (or produce warnings etc)

}


Expand All @@ -152,9 +151,8 @@ bool CFileHandler::TryReadFromVFS(const string& fileName, int section)
fileSize = fileBuffer.size();
return true;
}
#else
return false;
#endif
return false;
}


Expand Down
1 change: 1 addition & 0 deletions rts/System/FileSystem/FileHandler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#include <vector>
#include <string>
#include <fstream>
#include <memory>
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't this be added to some .cpp file(s) instead? Looks like no new code was added below so probably doesn't need to be in the header (gets applies everywhere the .h is included even if not all .cpp files need it)

#include <span>
#include <cinttypes>
#include <mio/mmap.hpp>
Expand Down
23 changes: 18 additions & 5 deletions rts/System/FileSystem/FileSystemAbstraction.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
#include <sys/stat.h>
#include <sys/types.h>
#include <cerrno>
#include <filesystem>
#include <chrono>
#include <fmt/printf.h>
#include <cstring>
#include "System/SpringRegex.h"
#include "System/Platform/Misc.h"
Expand Down Expand Up @@ -176,6 +179,7 @@ bool FileSystemAbstraction::IsReadableFile(const std::string& file)

unsigned int FileSystemAbstraction::GetFileModificationTime(const std::string& file)
{
#if 0
struct stat info;

if (stat(file.c_str(), &info) != 0) {
Expand All @@ -184,6 +188,17 @@ unsigned int FileSystemAbstraction::GetFileModificationTime(const std::string& f
}

return info.st_mtime;
#else
std::error_code ec;
auto fileTime = std::filesystem::last_write_time(file, ec);
if (ec) {
LOG_L(L_WARNING, "[FSA::%s] error '%s' getting last modification time of file '%s'", __func__, ec.message().c_str(), file.c_str());
return 0;
}

using namespace std::chrono;
return duration_cast<seconds>(file_clock::to_utc(fileTime).time_since_epoch()).count();
#endif
}

std::string FileSystemAbstraction::GetFileModificationDate(const std::string& file)
Expand All @@ -193,12 +208,10 @@ std::string FileSystemAbstraction::GetFileModificationDate(const std::string& fi
if (t == 0)
return "";

const struct tm* clk = std::gmtime(&t);
const char* fmt = "%d%02d%02d%02d%02d%02d";
//const struct tm* clk = std::gmtime(&t);
const auto* clk = std::localtime(&t);

char buf[67];
SNPRINTF(buf, sizeof(buf), fmt, 1900 + clk->tm_year, clk->tm_mon + 1, clk->tm_mday, clk->tm_hour, clk->tm_min, clk->tm_sec);
return buf;
return fmt::sprintf("%d%02d%02d%02d%02d%02d", 1900 + clk->tm_year, clk->tm_mon + 1, clk->tm_mday, clk->tm_hour, clk->tm_min, clk->tm_sec);
}


Expand Down
Loading