Skip to content

Commit

Permalink
MassDumper Optimization
Browse files Browse the repository at this point in the history 
Reduced Logging
  • Loading branch information
@GoneUp
GoneUp committed Jul 21, 2020
1 parent 1c63ecf commit b990b74
Show file tree
Hide file tree
Showing 6 changed files with 80 additions and 21 deletions.
27 changes: 21 additions & 6 deletions GPK_RePack/Forms/GUI.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ private void openToolStripMenuItem_Click(object sender, EventArgs e)
}

//display info while loading
while (!Task.WaitAll(runningTasks.ToArray(), 500))
while (!Task.WaitAll(runningTasks.ToArray(), 50))
{
Application.DoEvents();
DisplayStatus(runningReaders, "Loading", start);
Expand Down Expand Up @@ -344,9 +344,22 @@ private void DisplayStatus(List<IProgress> list, string tag, DateTime start)
foreach (IProgress p in list)
{
Status stat = p.GetStatus();
actual += stat.progress;
total += stat.totalobjects;
if (stat.finished) finished++;
if (stat.subGpkCount > 1)
{
//dont show actual objects, just the sub-file count
actual += stat.subGpkDone;
total += stat.subGpkCount;
if (actual == total) finished++;
}
else
{
//normal gpk
actual += stat.progress;
total += stat.totalobjects;
if (stat.finished) finished++;
}


}

if (finished < list.Count)
Expand Down Expand Up @@ -403,6 +416,7 @@ public void DrawPackages()
}
treeMain.BeginUpdate();
treeMain.Nodes.Clear();
var toAdd = new List<TreeNode>();

for (int i = 0; i < gpkStore.LoadedGpkPackages.Count; i++)
{
Expand Down Expand Up @@ -471,11 +485,12 @@ public void DrawPackages()

}

treeMain.Nodes.Add(nodeP);
toAdd.Add(nodeP);

}



treeMain.Nodes.AddRange(toAdd.ToArray());
treeMain.Sort();
treeMain.EndUpdate();
}
Expand Down
27 changes: 22 additions & 5 deletions GPK_RePack/IO/MassDumper.cs
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.ServiceModel.Configuration;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Windows;
using GPK_RePack.Editors;
Expand Down Expand Up @@ -103,23 +105,34 @@ public static void DumpMassHeaders(String outfile, String[] gpkFiles)
}
}


//in thread
public static void DumpMassTextures(GpkStore store, String outdir, Dictionary<String, List<CompositeMapEntry>> filterList)
{
logger.Info("Started dumping textures to " + outdir);
Directory.CreateDirectory(outdir);

List<Task> runningTasks = new List<Task>();
SynchronizedCollection<Task> runningTasks = new SynchronizedCollection<Task>();

int MAX_TASKS = 100;

foreach (var file in filterList)
{
//throttle thread creation
while (runningTasks.Count > MAX_TASKS)
{
Thread.Sleep(1000);
}


//create out dir
var fileOutPath = string.Format("{0}\\{1}.gpk\\", outdir, file.Key);
Directory.CreateDirectory(fileOutPath);

//limit to 5 threads by default
foreach (var entry in file.Value)
{
Task newTask = new Task(() =>
Task newTask = null;
newTask = new Task(() =>
{
string path = string.Format("{0}\\{1}.gpk", store.BaseSearchPath, entry.SubGPKName);

Expand Down Expand Up @@ -149,15 +162,19 @@ public static void DumpMassTextures(GpkStore store, String outdir, Dictionary<St
}

package = null;
runningTasks.Remove(newTask);
});


newTask.Start();
runningTasks.Add(newTask);
}


}
}

Task.WaitAll(runningTasks.ToArray());


NLogConfig.EnableFormLogging();
logger.Info("Dumping done");
Expand Down
11 changes: 7 additions & 4 deletions GPK_RePack/IO/Reader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public List<GpkPackage> ReadGpk(string path, bool skipExportData)

var fileID = string.Format("{0}_{1}", Path.GetFileName(path), reader.BaseStream.Position);
logger = LogManager.GetLogger("[Reader:" + fileID + "]");
logger.Info("Reading Start");
logger.Debug("Reading Start");

tmpGPK.Filename = fileID;
tmpGPK.Path = path;
Expand Down Expand Up @@ -74,15 +74,18 @@ public List<GpkPackage> ReadGpk(string path, bool skipExportData)

//header is read, let us extract the data and read it
List<GpkPackage> returnPackageList = new List<GpkPackage>();
stat = new Status();
stat.subGpkCount = tmpPackageList.Count;

foreach (var subGPK in tmpPackageList)
{
stat = new Status();

reader.BaseStream.Seek(subGPK.CompositeStartOffset, SeekOrigin.Begin);
var data = reader.ReadBytes((int)subGPK.OrginalSize);

var fullGpk = ReadSubGpkPackage(subGPK, data, skipExportData, stat);
returnPackageList.Add(fullGpk);

stat.subGpkDone++;
}

reader.Close();
Expand Down Expand Up @@ -253,7 +256,7 @@ private void ReadHeader(BinaryReader reader, GpkPackage package)
if (package.x64) package.Header.Unk3 = reader.ReadBytes(12);

stat.totalobjects = package.Header.NameCount + package.Header.ImportCount + package.Header.ExportCount * 3; //Export, Export Linking, ExportData = *3
logger.Info("File Info: NameCount {0}, ImportCount {1}, ExportCount {2}", package.Header.NameCount, package.Header.ImportCount, package.Header.ExportCount);
logger.Debug("File Info: NameCount {0}, ImportCount {1}, ExportCount {2}", package.Header.NameCount, package.Header.ImportCount, package.Header.ExportCount);

package.Header.FGUID = reader.ReadBytes(16);
//logger.Info("FGUID " + package.Header.FGUID);
Expand Down
3 changes: 3 additions & 0 deletions GPK_RePack/IO/Writer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ namespace GPK_RePack.IO
{
struct Status
{
public int subGpkCount;
public int subGpkDone;

public int progress;
public int totalobjects;
public long time;
Expand Down
16 changes: 15 additions & 1 deletion GPK_RePack/Model/Payload/MipMap.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ class MipMap

public byte[] uncompressedData;

public string loadedFromTextureCache = "";
public string textureCachePath = "";

public List<ChunkBlock> blocks = new List<ChunkBlock>();
internal int blockCount;
Expand All @@ -47,6 +47,10 @@ public void generateBlocks()
compressedSize = 0;
for (int i = 0; i < blockCount; i++)
{
//if data was not found, skip
if (uncompressedData == null)
continue;

int blockOffset = i * blocksize;
int blockEnd = blockOffset + blocksize;
if (blockEnd > uncompressedSize)
Expand All @@ -73,6 +77,16 @@ public override string ToString()
info.AppendFormat("Size: {0} x {1} {2}", sizeX, sizeY, Environment.NewLine);
info.AppendLine("Compression: " + flags);
if (((CompressionTypes)flags & NoOp) != 0) info.AppendLine("Data for this MipMap is stored external!");
if (((CompressionTypes)flags & CompressionTypes.StoreInSeparatefile) != 0)
{
info.AppendLine("Data for this MipMap is stored in tfc");
if (compressedSize == 0)
{
info.AppendLine("TFC data was not found. Searched for tfc file at " + textureCachePath);
}
}


info.AppendLine("Compressed Size: " + compressedSize);
info.AppendLine("Uncompressed Size: " + uncompressedSize);
info.AppendLine("Blocks: " + blocks.Count);
Expand Down
17 changes: 12 additions & 5 deletions GPK_RePack/Model/Payload/Texture2D.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,11 +136,15 @@ public void ReadData(GpkPackage package, GpkExport export)
{
tgaPath = Reader.ReadString(reader, length);
}
else
else if (length < 0)
{
inUnicode = true;
tgaPath = Reader.ReadUnicodeString(reader, (length * -1) * 2);
}
else
{
tgaPath = "";
}


int count = reader.ReadInt32();
Expand Down Expand Up @@ -169,13 +173,15 @@ public void ReadData(GpkPackage package, GpkExport export)
//compChunkSize == size
//TextureFileCacheName prop has name
var txtProp = export.Properties.Find(t => ((GpkBaseProperty)t).name == "TextureFileCacheName");
if (txtProp == null)
goto fail;
String txtCacheFile = ((GpkNameProperty)txtProp).value;

//assumption: cache in same dir, happens for cookedpc compositegpks
var path = $"{Path.GetDirectoryName(package.Path)}\\{txtCacheFile}.tfc";
if (File.Exists(path))
map.textureCachePath = $"{Path.GetDirectoryName(package.Path)}\\{txtCacheFile}.tfc";
if (File.Exists(map.textureCachePath))
{
BinaryReader cacheReader = new BinaryReader(new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read));
BinaryReader cacheReader = new BinaryReader(new FileStream(map.textureCachePath, FileMode.Open, FileAccess.Read, FileShare.Read));
cacheReader.BaseStream.Seek(map.compChunkOffset, SeekOrigin.Begin);

map.signature = cacheReader.ReadUInt32(); //0x9e2a83c1
Expand All @@ -189,7 +195,7 @@ public void ReadData(GpkPackage package, GpkExport export)
}
else
{
logger.Warn("{0}, MipMap {1}, Cache {2}, CompressionTypes.StoreInSeparatefile, could not find tfc!!", export.ObjectName, i, txtCacheFile);
logger.Debug("{0}, MipMap {1}, Cache {2}, CompressionTypes.StoreInSeparatefile, could not find tfc!!", export.ObjectName, i, txtCacheFile);
}
}
else if (((CompressionTypes)map.flags & CompressionTypes.SeperateData) != 0)
Expand All @@ -207,6 +213,7 @@ public void ReadData(GpkPackage package, GpkExport export)
logger.Trace("{0}, MipMap {0}, no data!!", export.ObjectName, i);
}

fail:
map.sizeX = reader.ReadInt32();
map.sizeY = reader.ReadInt32();

Expand Down

0 comments on commit b990b74

Please sign in to comment.