Skip to content

Commit

Permalink
Add Queues to E2 File Extension (#2763)
Browse files Browse the repository at this point in the history 
* File conversion to queue + fixes:
- Add queuing for file load operations
- Added cvar wire_expression2_file_max_queue to limit number of queued files
- Added event fileList(sr) to eventify fileListClk
- Added file<Load/Write/List>Queued() to return number of queued elements
- Added outputs to various functions for debugging
- Added various exceptions
- Increased file Load/Write/List op cost to 100 (from 20)
- Decreased op costs of certain functions
- Fixes runOnFile triggering when runOnFile is not set
- Fixes runOnList triggering when runOnFileList is not set
- Deprecate runOn*- and *Clk-type functions
- Removed some previously useless code
- Removed old delay system

* lints

* Reduced network strings
Optimize some small things
Fixed backwards *Queue functions

* Fix comment

* Add fileWritten event

* Fix typo
Avoid race condition with event fileWritten
  • Loading branch information
@Denneisk
Denneisk authored Nov 2, 2023
1 parent d9a46cc commit 12e109e
Show file tree
Hide file tree
Showing 3 changed files with 395 additions and 284 deletions.
117 changes: 63 additions & 54 deletions lua/entities/gmod_wire_expression2/core/cl_files.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,14 @@
By: Dan (McLovin)
]]--

local cv_max_transfer_size = CreateConVar( "wire_expression2_file_max_size", "300", { FCVAR_REPLICATED, FCVAR_ARCHIVE } ) //in kib
local cv_max_transfer_size = CreateConVar("wire_expression2_file_max_size", "300", { FCVAR_REPLICATED, FCVAR_ARCHIVE }, "Maximum file size in kibibytes.")

local upload_buffer = {}
local download_buffer = {}

local upload_chunk_size = 20000 //Our overhead is pretty small so lets send it in moderate sized pieces, no need to max out the buffer
local upload_chunk_size = 20000 --Our overhead is pretty small so lets send it in moderate sized pieces, no need to max out the buffer

local allowed_directories = { //prefix with >(allowed directory)/file.txt for files outside of e2files/ directory
local allowed_directories = { --prefix with >(allowed directory)/file.txt for files outside of e2files/ directory
["e2files"] = "e2files",
["e2shared"] = "expression2/e2shared",
["cpushared"] = "cpuchip/e2shared",
Expand Down Expand Up @@ -47,88 +47,92 @@ local function process_filepath( filepath )
return string.GetPathFromFilename( fullpath ) or "e2files/", string.GetFileFromFilename( fullpath ) or "noname.txt"
end

/* --- File Read --- */
--- File Read ---

local function upload_callback()
if not upload_buffer or not upload_buffer.data then return end

local chunk_size = math.Clamp( string.len( upload_buffer.data ), 0, upload_chunk_size )
local chunk_size = math.Clamp(#upload_buffer.data, 0, upload_chunk_size)

net.Start("wire_expression2_file_chunk")
net.Start("wire_expression2_file_upload")
net.WriteUInt(2, 2)
net.WriteUInt(chunk_size, 32)
net.WriteData(upload_buffer.data, chunk_size)
net.SendToServer()
upload_buffer.data = string.sub( upload_buffer.data, chunk_size + 1, string.len( upload_buffer.data ) )
upload_buffer.data = string.sub(upload_buffer.data, chunk_size + 1)

if upload_buffer.chunk >= upload_buffer.chunks then
net.Start("wire_expression2_file_finish") net.SendToServer()
net.Start("wire_expression2_file_upload") net.WriteUInt(3, 2) net.SendToServer()
timer.Remove( "wire_expression2_file_upload" )
return
end

upload_buffer.chunk = upload_buffer.chunk + 1
end

net.Receive("wire_expression2_request_file_sp", function(netlen)
local fpath,fname = process_filepath(net.ReadString())
RunConsoleCommand("wire_expression2_file_singleplayer", fpath .. fname)
end)

net.Receive("wire_expression2_request_file", function(netlen)
local fpath,fname = process_filepath(net.ReadString())
local fullpath = fpath .. fname
if net.ReadBool() then
RunConsoleCommand("wire_expression2_file_singleplayer", fpath .. fname)
else
local fullpath = fpath .. fname

if file.Exists( fullpath,"DATA" ) and file.Size( fullpath, "DATA" ) <= (cv_max_transfer_size:GetInt() * 1024) then
local filedata = file.Read( fullpath,"DATA" ) or ""
if file.Exists( fullpath,"DATA" ) and file.Size( fullpath, "DATA" ) <= (cv_max_transfer_size:GetInt() * 1024) then
local filedata = file.Read( fullpath,"DATA" ) or ""

local len = #filedata
local len = #filedata

upload_buffer = {
chunk = 1,
chunks = math.ceil(len / upload_chunk_size),
data = filedata
}
upload_buffer = {
chunk = 1,
chunks = math.ceil(len / upload_chunk_size),
data = filedata
}

net.Start("wire_expression2_file_begin")
net.WriteUInt(len, 32)
net.SendToServer()
net.Start("wire_expression2_file_upload")
net.WriteUInt(1, 2)
net.WriteUInt(len, 32)
net.SendToServer()

timer.Create( "wire_expression2_file_upload", 1/60, upload_buffer.chunks, upload_callback )
else
net.Start("wire_expression2_file_begin")
net.WriteUInt(0, 32) // 404 file not found, send len of 0
net.SendToServer()
timer.Create( "wire_expression2_file_upload", 1/60, upload_buffer.chunks, upload_callback )
else
net.Start("wire_expression2_file_upload")
net.WriteUInt(1, 2)
net.WriteUInt(0, 32) -- 404 file not found, send len of 0
net.SendToServer()
end
end
end )

/* --- File Write --- */
net.Receive("wire_expression2_file_download_begin", function( netlen )
local fpath,fname = process_filepath( net.ReadString() )
if not E2Lib.isValidFileWritePath(fname) then return end
if not file.Exists(fpath, "DATA") then file.CreateDir(fpath) end
download_buffer = {
name = fpath .. fname,
data = ""
}
end )

net.Receive("wire_expression2_file_download_chunk", function( netlen )
if not download_buffer.name then return end
local len = net.ReadUInt(32)
download_buffer.data = (download_buffer.data or "") .. net.ReadData(len)
end )

net.Receive("wire_expresison2_file_download_finish", function( netlen )
if not download_buffer.name then return end
--- File Write ---
net.Receive("wire_expression2_file_download", function()
local flag = net.ReadUInt(2)
if flag == 2 then -- Chunk
if not download_buffer.name then return end
local len = net.ReadUInt(32)
download_buffer.data = (download_buffer.data or "") .. net.ReadData(len)

elseif flag == 1 then -- Begin
local fpath,fname = process_filepath( net.ReadString() )
if not E2Lib.isValidFileWritePath(fname) then return end
if not file.Exists(fpath, "DATA") then file.CreateDir(fpath) end
download_buffer = {
name = fpath .. fname,
data = ""
}
elseif flag == 3 then -- End
if not download_buffer.name then return end

if net.ReadBit() ~= 0 then
file.Append( download_buffer.name, download_buffer.data )
else
file.Write( download_buffer.name, download_buffer.data )
if net.ReadBool() then
file.Append( download_buffer.name, download_buffer.data )
else
file.Write( download_buffer.name, download_buffer.data )
end
else -- Abort
download_buffer = {}
end
end )

/* --- File List --- */
--- File List ---

net.Receive( "wire_expression2_request_list", function( netlen )
local dir = process_filepath(net.ReadString())
Expand All @@ -148,3 +152,8 @@ net.Receive( "wire_expression2_request_list", function( netlen )
end
net.SendToServer()
end )

net.Receive("wire_expression2_file_abort", function()
timer.Remove("wire_expression2_file_upload")
print("Aborted")
end)
Loading

0 comments on commit 12e109e

Please sign in to comment.