From 10bb7182790fcebefa1b28bd6e47d2f3d35b3582 Mon Sep 17 00:00:00 2001 From: GoneUp Date: Thu, 16 Jul 2020 00:02:18 +0200 Subject: [PATCH] Less logging, memory leak fixes, memory optimization --- GPK_RePack/Forms/GUI.Designer.cs | 39 +++++++------- GPK_RePack/Forms/GUI.cs | 5 -- GPK_RePack/Forms/NLogConfig.cs | 7 +-- GPK_RePack/IO/MassDumper.cs | 1 + GPK_RePack/IO/Reader.cs | 29 ++++++++--- GPK_RePack/Model/GpkPackage.cs | 3 +- GPK_RePack/Model/Payload/MipMap.cs | 1 + GPK_RePack/Model/Payload/Texture2D.cs | 75 +++++++++++++++------------ UpkManager.Dds/DdsFile.cs | 1 - 9 files changed, 91 insertions(+), 70 deletions(-) diff --git a/GPK_RePack/Forms/GUI.Designer.cs b/GPK_RePack/Forms/GUI.Designer.cs index 34c31a7..f7f9254 100644 --- a/GPK_RePack/Forms/GUI.Designer.cs +++ b/GPK_RePack/Forms/GUI.Designer.cs @@ -65,6 +65,7 @@ private void InitializeComponent() this.compositeGPKToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.decryptionToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.loadMappingToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); + this.dumpCompositeTexturesToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.miscToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.setFilesizeToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.setAllPropertysToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); @@ -110,14 +111,13 @@ private void InitializeComponent() this.btnCopy = new System.Windows.Forms.Button(); this.btnDelete = new System.Windows.Forms.Button(); this.btnPaste = new System.Windows.Forms.Button(); - this.boxLog = new System.Windows.Forms.TextBox(); this.splitContainerTreeInfo = new System.Windows.Forms.SplitContainer(); this.statusStrip = new System.Windows.Forms.StatusStrip(); this.lblFiller = new System.Windows.Forms.ToolStripStatusLabel(); this.lblStatus = new System.Windows.Forms.ToolStripStatusLabel(); this.ProgressBar = new System.Windows.Forms.ToolStripProgressBar(); this.splitContainerLog_InfoTree = new System.Windows.Forms.SplitContainer(); - this.dumpCompositeTexturesToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); + this.boxLog = new System.Windows.Forms.RichTextBox(); this.treeContextMenu.SuspendLayout(); this.menuStrip.SuspendLayout(); this.tabControl.SuspendLayout(); @@ -398,6 +398,13 @@ private void InitializeComponent() this.loadMappingToolStripMenuItem.Text = "Load Mapping"; this.loadMappingToolStripMenuItem.Click += new System.EventHandler(this.loadMappingToolStripMenuItem_Click); // + // dumpCompositeTexturesToolStripMenuItem + // + this.dumpCompositeTexturesToolStripMenuItem.Name = "dumpCompositeTexturesToolStripMenuItem"; + this.dumpCompositeTexturesToolStripMenuItem.Size = new System.Drawing.Size(236, 24); + this.dumpCompositeTexturesToolStripMenuItem.Text = "Dump CompositeTextures"; + this.dumpCompositeTexturesToolStripMenuItem.Click += new System.EventHandler(this.dumpCompositeTexturesToolStripMenuItem_Click); + // // miscToolStripMenuItem // this.miscToolStripMenuItem.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] { @@ -895,18 +902,6 @@ private void InitializeComponent() this.btnPaste.UseVisualStyleBackColor = true; this.btnPaste.Click += new System.EventHandler(this.btnPaste_Click); // - // boxLog - // - this.boxLog.Dock = System.Windows.Forms.DockStyle.Fill; - this.boxLog.Location = new System.Drawing.Point(0, 0); - this.boxLog.Multiline = true; - this.boxLog.Name = "boxLog"; - this.boxLog.ReadOnly = true; - this.boxLog.ScrollBars = System.Windows.Forms.ScrollBars.Vertical; - this.boxLog.Size = new System.Drawing.Size(1163, 125); - this.boxLog.TabIndex = 5; - this.boxLog.TextChanged += new System.EventHandler(this.boxLog_TextChanged); - // // splitContainerTreeInfo // this.splitContainerTreeInfo.Dock = System.Windows.Forms.DockStyle.Fill; @@ -975,12 +970,15 @@ private void InitializeComponent() this.splitContainerLog_InfoTree.SplitterDistance = 522; this.splitContainerLog_InfoTree.TabIndex = 9; // - // dumpCompositeTexturesToolStripMenuItem + // boxLog // - this.dumpCompositeTexturesToolStripMenuItem.Name = "dumpCompositeTexturesToolStripMenuItem"; - this.dumpCompositeTexturesToolStripMenuItem.Size = new System.Drawing.Size(236, 24); - this.dumpCompositeTexturesToolStripMenuItem.Text = "Dump CompositeTextures"; - this.dumpCompositeTexturesToolStripMenuItem.Click += new System.EventHandler(this.dumpCompositeTexturesToolStripMenuItem_Click); + this.boxLog.Dock = System.Windows.Forms.DockStyle.Fill; + this.boxLog.Location = new System.Drawing.Point(0, 0); + this.boxLog.Name = "boxLog"; + this.boxLog.ReadOnly = true; + this.boxLog.Size = new System.Drawing.Size(1163, 125); + this.boxLog.TabIndex = 0; + this.boxLog.Text = ""; // // GUI // @@ -1023,7 +1021,6 @@ private void InitializeComponent() this.statusStrip.PerformLayout(); this.splitContainerLog_InfoTree.Panel1.ResumeLayout(false); this.splitContainerLog_InfoTree.Panel2.ResumeLayout(false); - this.splitContainerLog_InfoTree.Panel2.PerformLayout(); ((System.ComponentModel.ISupportInitialize)(this.splitContainerLog_InfoTree)).EndInit(); this.splitContainerLog_InfoTree.ResumeLayout(false); this.ResumeLayout(false); @@ -1042,7 +1039,6 @@ private void InitializeComponent() private System.Windows.Forms.TabControl tabControl; private System.Windows.Forms.TabPage tabInfo; private System.Windows.Forms.ToolStripSeparator toolStripSeparator1; - private System.Windows.Forms.TextBox boxLog; private System.Windows.Forms.TextBox boxInfo; private System.Windows.Forms.Button btnImportRaw; private System.Windows.Forms.Button btnExportRaw; @@ -1119,6 +1115,7 @@ private void InitializeComponent() private ToolStripMenuItem decryptionToolStripMenuItem; private ToolStripMenuItem loadMappingToolStripMenuItem; private ToolStripMenuItem dumpCompositeTexturesToolStripMenuItem; + private RichTextBox boxLog; } } diff --git a/GPK_RePack/Forms/GUI.cs b/GPK_RePack/Forms/GUI.cs index 1459469..96e58ea 100644 --- a/GPK_RePack/Forms/GUI.cs +++ b/GPK_RePack/Forms/GUI.cs @@ -146,11 +146,6 @@ private void exitToolStripMenuItem_Click(object sender, EventArgs e) Environment.Exit(0); } - private void boxLog_TextChanged(object sender, EventArgs e) - { - boxLog.SelectionStart = boxLog.TextLength; - boxLog.ScrollToCaret(); - } private void clearToolStripMenuItem_Click(object sender, EventArgs e) { diff --git a/GPK_RePack/Forms/NLogConfig.cs b/GPK_RePack/Forms/NLogConfig.cs index 317b4b0..808652c 100644 --- a/GPK_RePack/Forms/NLogConfig.cs +++ b/GPK_RePack/Forms/NLogConfig.cs @@ -29,11 +29,12 @@ public static void SetDefaultConfig() AsyncTargetWrapper asyncWrapperLog = new AsyncTargetWrapper(logfile); config.AddTarget("logfile", asyncWrapperLog); - var formTargetSync = new FormControlTarget(); - formTargetSync.Layout = "${date:format=HH\\:mm\\:ss} ${logger} # ${message} ${newline}"; - formTargetSync.Append = true; + var formTargetSync = new RichTextBoxTarget(); + formTargetSync.Layout = "${date:format=HH\\:mm\\:ss} ${logger} # ${message}"; + formTargetSync.AutoScroll = true; formTargetSync.ControlName = "boxLog"; formTargetSync.FormName = "GUI"; + formTargetSync.MaxLines = 1000; config.AddTarget("form", formTargetSync); formTarget = new AsyncTargetWrapper(formTargetSync); config.AddTarget("form", formTarget); diff --git a/GPK_RePack/IO/MassDumper.cs b/GPK_RePack/IO/MassDumper.cs index 6fdc3e0..0469527 100644 --- a/GPK_RePack/IO/MassDumper.cs +++ b/GPK_RePack/IO/MassDumper.cs @@ -129,6 +129,7 @@ public static void DumpMassTextures(GpkStore store, String outdir) Reader r = new Reader(); var package = r.ReadSubGpkFromComposite(path, entry.UID, entry.FileOffset, entry.FileLength); + package.LowMemMode = true; //extract var exports = package.GetExportsByClass("Core.Texture2D"); diff --git a/GPK_RePack/IO/Reader.cs b/GPK_RePack/IO/Reader.cs index 83eeee6..76e74a1 100644 --- a/GPK_RePack/IO/Reader.cs +++ b/GPK_RePack/IO/Reader.cs @@ -106,16 +106,19 @@ public List ReadGpk(string path, bool skipExportData) public GpkPackage ReadSubGpkFromComposite(string path, string fileID, int fileOffset, int dataLength) { + BinaryReader reader = null; + try { logger = LogManager.GetLogger("ReadSubGpkFromComposite: " + fileID); - var reader = new BinaryReader(new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)); + reader = new BinaryReader(new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)); stat = new Status(); reader.BaseStream.Seek(fileOffset, SeekOrigin.Begin); var data = reader.ReadBytes(dataLength); reader.Close(); + reader.Dispose(); GpkPackage tmpGPK = new GpkPackage(); tmpGPK.Filename = fileID; @@ -127,7 +130,7 @@ public GpkPackage ReadSubGpkFromComposite(string path, string fileID, int fileOf var fullGpk = ReadSubGpkPackage(tmpGPK, data, false, stat); - logger.Debug("done"); + logger.Debug("Done"); return fullGpk; } @@ -136,19 +139,26 @@ public GpkPackage ReadSubGpkFromComposite(string path, string fileID, int fileOf logger.Fatal("Parse failure!"); logger.Fatal(ex); } + finally + { + if (reader != null) + reader.Close(); + } return null; } private GpkPackage ReadSubGpkPackage(GpkPackage package, byte[] data, bool skipExportData, Status stat) { + BinaryReader reader = null; + try { - BinaryReader reader = new BinaryReader(new MemoryStream(data)); + reader = new BinaryReader(new MemoryStream(data)); Stopwatch pkgWatch = new Stopwatch(); logger = LogManager.GetLogger("[ReadSubGpkPackage:" + package.Filename + "]"); - logger.Info("Reading Start"); + logger.Debug("Reading Start"); stat.name = package.Filename; pkgWatch.Start(); @@ -158,6 +168,7 @@ private GpkPackage ReadSubGpkPackage(GpkPackage package, byte[] data, bool skipE if (file != null) { reader.Close(); + reader.Dispose(); reader = new BinaryReader(new MemoryStream(file)); } @@ -175,7 +186,6 @@ private GpkPackage ReadSubGpkPackage(GpkPackage package, byte[] data, bool skipE stat.time = pkgWatch.ElapsedMilliseconds; stat.finished = true; logger.Info("Reading of package {0} complete, took {1}ms!", package.Filename, pkgWatch.ElapsedMilliseconds); - logger.Info("Reading Done"); return package; } @@ -184,6 +194,11 @@ private GpkPackage ReadSubGpkPackage(GpkPackage package, byte[] data, bool skipE logger.Fatal("Parse failure!"); logger.Fatal(ex); } + finally + { + if (reader != null) + reader.Close(); + } return null; } @@ -221,7 +236,7 @@ private void ReadHeader(BinaryReader reader, GpkPackage package) package.Header.DependsOffset = reader.ReadInt32(); - if (package.x64) package.Header.HeaderSize = reader.ReadInt32(); + if (package.x64) package.Header.HeaderSize = reader.ReadInt32(); logger.Debug("NameCount " + package.Header.NameCount); logger.Debug("NameOffset " + package.Header.NameOffset); @@ -570,7 +585,7 @@ private void ReadExportData(BinaryReader reader, GpkPackage package) if (export.Payload != null) logger.Debug(export.Payload.ToString()); } - } + } logger.Trace(String.Format("Export {0}: Read Data ({1} bytes {2}) and {3} Properties ({4} bytes)", export.ObjectName, toread, tag, export.Properties.Count, export.PropertySize)); diff --git a/GPK_RePack/Model/GpkPackage.cs b/GPK_RePack/Model/GpkPackage.cs index 7378a45..7dce5b1 100644 --- a/GPK_RePack/Model/GpkPackage.cs +++ b/GPK_RePack/Model/GpkPackage.cs @@ -25,8 +25,9 @@ class GpkPackage public long OrginalSize; //raw compressed size public long UncompressedSize; - + // public Boolean Changes = false; + public Boolean LowMemMode = false; //data structs public GpkHeader Header; diff --git a/GPK_RePack/Model/Payload/MipMap.cs b/GPK_RePack/Model/Payload/MipMap.cs index d2ad86c..f2c6c1b 100644 --- a/GPK_RePack/Model/Payload/MipMap.cs +++ b/GPK_RePack/Model/Payload/MipMap.cs @@ -37,6 +37,7 @@ class MipMap public string loadedFromTextureCache = ""; public List blocks = new List(); + internal int blockCount; public void generateBlocks() { diff --git a/GPK_RePack/Model/Payload/Texture2D.cs b/GPK_RePack/Model/Payload/Texture2D.cs index 0ec56cc..7457950 100644 --- a/GPK_RePack/Model/Payload/Texture2D.cs +++ b/GPK_RePack/Model/Payload/Texture2D.cs @@ -27,6 +27,7 @@ class Texture2D : IPayload public bool inUnicode = false; public byte[] guid; + private const CompressionTypes NothingToDo = CompressionTypes.Unused | CompressionTypes.StoreInSeparatefile; public List maps = new List(); @@ -49,13 +50,14 @@ public void WriteData(BinaryWriter writer, GpkPackage package, GpkExport export) Writer.WriteString(writer, tgaPath, true); } - + writer.Write(maps.Count); foreach (var map in maps) { //refressh block info, compress blocks - //map.generateBlocks(); + if (map.blocks.Count == 0) + map.generateBlocks(); //chunk //info @@ -184,7 +186,8 @@ public void ReadData(GpkPackage package, GpkExport export) } cacheReader.Close(); - } else + } + else { logger.Warn("{0}, MipMap {1}, Cache {2}, CompressionTypes.StoreInSeparatefile, could not find tfc!!", export.ObjectName, i, txtCacheFile); } @@ -214,7 +217,7 @@ public void ReadData(GpkPackage package, GpkExport export) guid = reader.ReadBytes(16); } - private void ReadMipMapFromReader(BinaryReader reader, MipMap map) + private void ReadMipMapFromReader(BinaryReader reader, MipMap map, GpkPackage package) { map.signature = reader.ReadUInt32(); //0x9e2a83c1 Debug.Assert(map.signature == MipMap.DEFAULT_SIGNATURE); @@ -225,11 +228,11 @@ private void ReadMipMapFromReader(BinaryReader reader, MipMap map) map.uncompressedSize_chunkheader = reader.ReadInt32(); map.uncompressedData = new byte[map.uncompressedSize]; - int blockCount = (map.uncompressedSize + map.blocksize - 1) / map.blocksize; + map.blockCount = (map.uncompressedSize + map.blocksize - 1) / map.blocksize; int blockOffset = 0; - for (int j = 0; j < blockCount; ++j) + for (int j = 0; j < map.blockCount; ++j) { var block = new ChunkBlock(); block.compressedSize = reader.ReadInt32(); @@ -250,6 +253,13 @@ private void ReadMipMapFromReader(BinaryReader reader, MipMap map) //save memory block.uncompressedData = null; } + + //we clear the compressed chunkblocks, so only uncompressed data is present + //blocks need to be rebuild when saving + if (package.LowMemMode) + { + map.blocks.Clear(); + } } @@ -264,7 +274,7 @@ public int GetSize() { //header tmpSize += 32; - tmpSize += map.blocks.Count * 8 + map.compressedSize; + tmpSize += map.blockCount * 8 + map.compressedSize; //sizex, sizey tmpSize += 8; } @@ -305,52 +315,53 @@ public Stream GetObjectStream() { if (maps == null || !maps.Any()) return null; - FileFormat format; - MipMap mipMap = maps.Where(mm => mm.uncompressedData != null && mm.uncompressedData.Length > 0).OrderByDescending(mm => mm.sizeX > mm.sizeY ? mm.sizeX : mm.sizeY).FirstOrDefault(); + if (mipMap == null) + return null; - return mipMap == null ? null : buildDdsImage(maps.IndexOf(mipMap), out format); + return buildDdsImage(maps.IndexOf(mipMap)); } - private Stream buildDdsImage(int mipMapIndex, out FileFormat imageFormat) + private Stream buildDdsImage(int mipMapIndex) { MipMap mipMap = maps[mipMapIndex]; - - imageFormat = GetFormat(); - DdsHeader ddsHeader = new DdsHeader(new DdsSaveConfig(imageFormat, 0, 0, false, false), mipMap.sizeX, mipMap.sizeY); + DdsHeader ddsHeader = new DdsHeader(new DdsSaveConfig(GetFormat(), 0, 0, false, false), mipMap.sizeX, mipMap.sizeY); MemoryStream stream = new MemoryStream(); BinaryWriter writer = new BinaryWriter(stream); ddsHeader.Write(writer); - stream.Write(mipMap.uncompressedData, 0, mipMap.uncompressedData.Length); - stream.Flush(); - stream.Position = 0; + writer.Write(mipMap.uncompressedData); + writer.Flush(); + writer.BaseStream.Position = 0; return stream; } public void SaveObject(string filename, object configuration) { - if (maps == null || !maps.Any()) return; - - DdsSaveConfig config = configuration as DdsSaveConfig ?? new DdsSaveConfig(FileFormat.Unknown, 0, 0, false, false); - FileFormat format; - - MipMap mipMap = maps.Where(mm => mm.uncompressedData != null && mm.uncompressedData.Length > 0).OrderByDescending(mm => mm.sizeX > mm.sizeY ? mm.sizeX : mm.sizeY).FirstOrDefault(); - if (mipMap == null) return; + FileStream ddsStream = null; + try + { + if (maps == null || !maps.Any()) return; - Stream memory = buildDdsImage(maps.IndexOf(mipMap), out format); - if (memory == null) return; + DdsSaveConfig config = configuration as DdsSaveConfig ?? new DdsSaveConfig(FileFormat.Unknown, 0, 0, false, false); + config.FileFormat = GetFormat(); - DdsFile ddsImage = new DdsFile(GetObjectStream()); - FileStream ddsStream = new FileStream(filename, FileMode.Create); + //parse uncompressed image + DdsFile ddsImage = new DdsFile(GetObjectStream()); - config.FileFormat = format; - ddsImage.Save(ddsStream, config); - ddsStream.Close(); + ddsStream = new FileStream(filename, FileMode.Create); + ddsImage.Save(ddsStream, config); - memory.Close(); + ddsStream.Close(); + ddsStream.Dispose(); + } + finally + { + if (ddsStream != null) + ddsStream.Dispose(); + } } } diff --git a/UpkManager.Dds/DdsFile.cs b/UpkManager.Dds/DdsFile.cs index 2fb5c5c..693ce9f 100644 --- a/UpkManager.Dds/DdsFile.cs +++ b/UpkManager.Dds/DdsFile.cs @@ -377,7 +377,6 @@ public void Save(Stream output, DdsSaveConfig saveConfig) BinaryWriter writer = new BinaryWriter(output); header = new DdsHeader(saveConfig, Width, Height); - header.Write(writer); if (saveConfig.GenerateMipMaps) GenerateMipMaps();