From 800db3f112f1391a4c087b6614ee6fd8a29d5624 Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 21 Jan 2020 10:53:23 -0500 Subject: [PATCH 1/8] feat: Adds in ParseOption classes This class along with its associated code allows the users to set different options for parsing. Some default options are already defined (minimal, typical, and full) The purpose of this change is to reduce cycle time of the parser parsing data that is not used by the end user. Things like mouse events and messages are removed from the Typical ParseOptions option. Some minor syntax changes. These changes were originally written by @poma and @martijnhoekstra and consolidated by @zemill --- .../Program.cs | 2 +- Heroes.ReplayParser/BitReader.cs | 68 +++--- Heroes.ReplayParser/DataParser.cs | 127 ++++++----- .../MPQFiles/ReplayGameEvents.cs | 199 ++++++++++-------- Heroes.ReplayParser/ParseOptions.cs | 56 +++++ 5 files changed, 281 insertions(+), 171 deletions(-) create mode 100644 Heroes.ReplayParser/ParseOptions.cs diff --git a/Heroes.ReplayParser.ConsoleApplication/Program.cs b/Heroes.ReplayParser.ConsoleApplication/Program.cs index fe937c1..8d8177c 100644 --- a/Heroes.ReplayParser.ConsoleApplication/Program.cs +++ b/Heroes.ReplayParser.ConsoleApplication/Program.cs @@ -15,7 +15,7 @@ static void Main(string[] args) // Attempt to parse the replay // Ignore errors can be set to true if you want to attempt to parse currently unsupported replays, such as 'VS AI' or 'PTR Region' replays - var replayParseResult = DataParser.ParseReplay(randomReplayFileName, ignoreErrors: false, deleteFile: false); + var replayParseResult = DataParser.ParseReplay(randomReplayFileName, deleteFile: false, ParseOptions.TypicalParsing); // If successful, the Replay object now has all currently available information if (replayParseResult.Item1 == DataParser.ReplayParseResult.Success) diff --git a/Heroes.ReplayParser/BitReader.cs b/Heroes.ReplayParser/BitReader.cs index b6321a3..cbb92c4 100644 --- a/Heroes.ReplayParser/BitReader.cs +++ b/Heroes.ReplayParser/BitReader.cs @@ -20,7 +20,7 @@ public class BitReader public BitReader(Stream stream) { this.stream = stream; - this.Cursor = 0; + Cursor = 0; } /// @@ -35,7 +35,7 @@ public bool EndOfStream { get { - return (this.Cursor >> 3) == this.stream.Length; + return (Cursor >> 3) == stream.Length; } } @@ -62,31 +62,54 @@ public uint Read(int numBits) while (numBits > 0) { - var bytePos = this.Cursor & 7; + var bytePos = Cursor & 7; int bitsLeftInByte = 8 - bytePos; if (bytePos == 0) { - this.currentByte = this.stream.ReadByte(); + currentByte = stream.ReadByte(); } var bitsToRead = (bitsLeftInByte > numBits) ? numBits : bitsLeftInByte; - value = (value << bitsToRead) | ((uint)this.currentByte >> bytePos) & ((1u << bitsToRead) - 1u); - this.Cursor += bitsToRead; + value = (value << bitsToRead) | ((uint)currentByte >> bytePos) & ((1u << bitsToRead) - 1u); + Cursor += bitsToRead; numBits -= bitsToRead; } return value; } + /// + /// Skip specified number of bits in stream. + /// + /// The number of bits to skip. + public void Skip(int numBits) + { + // todo: calculade number of bytes to skip and just increment this.stream position + while (numBits > 0) + { + var bytePos = Cursor & 7; + int bitsLeftInByte = 8 - bytePos; + if (bytePos == 0) + { + currentByte = stream.ReadByte(); + } + + var bitsToRead = (bitsLeftInByte > numBits) ? numBits : bitsLeftInByte; + + Cursor += bitsToRead; + numBits -= bitsToRead; + } + } + /// /// If in the middle of a byte, moves to the start of the next byte. /// public void AlignToByte() { - if ((this.Cursor & 7) > 0) + if ((Cursor & 7) > 0) { - this.Cursor = (this.Cursor & 0x7ffffff8) + 8; + Cursor = (Cursor & 0x7ffffff8) + 8; } } @@ -95,10 +118,7 @@ public void AlignToByte() /// /// Number of bits to read, up to 32. /// Returns a uint containing the number of bits read. - public uint Read(uint numBits) - { - return this.Read((int)numBits); - } + public uint Read(uint numBits) => Read((int)numBits); public bool[] ReadBitArray(uint numBits) { @@ -115,10 +135,7 @@ public bool[] ReadBitArray(uint numBits) /// /// The . /// - public byte ReadByte() - { - return (byte)this.Read(8); - } + public byte ReadByte() => (byte)Read(8); /// /// Reads a single bit from the stream as a boolean. @@ -126,10 +143,7 @@ public byte ReadByte() /// /// The . /// - public bool ReadBoolean() - { - return this.Read(1) == 1; - } + public bool ReadBoolean() => Read(1) == 1; /// /// Reads 2 bytes from the stream as a short. @@ -137,10 +151,7 @@ public bool ReadBoolean() /// /// The . /// - public short ReadInt16() - { - return (short)this.Read(16); - } + public short ReadInt16() => (short)Read(16); /// /// Reads 4 bytes from the stream as an int. @@ -148,10 +159,7 @@ public short ReadInt16() /// /// The . /// - public int ReadInt32() - { - return (int)this.Read(32); - } + public int ReadInt32() => (int)Read(32); /// /// Reads an array of bytes from the stream. @@ -167,7 +175,7 @@ public byte[] ReadBytes(int bytes) var buffer = new byte[bytes]; for (int i = 0; i < bytes; i++) { - buffer[i] = this.ReadByte(); + buffer[i] = ReadByte(); } return buffer; @@ -184,7 +192,7 @@ public byte[] ReadBytes(int bytes) /// public string ReadString(int length) { - var buffer = this.ReadBytes(length); + var buffer = ReadBytes(length); return Encoding.UTF8.GetString(buffer); } diff --git a/Heroes.ReplayParser/DataParser.cs b/Heroes.ReplayParser/DataParser.cs index 327e079..bfdf2fb 100644 --- a/Heroes.ReplayParser/DataParser.cs +++ b/Heroes.ReplayParser/DataParser.cs @@ -39,7 +39,8 @@ public enum ReplayParseResult { "Battlefield of Eternity", new Tuple(-5.0, 33.0, 1.09, 0.96) } }; - public static Tuple ParseReplay(byte[] bytes, bool ignoreErrors = false, bool allowPTRRegion = false) + //public static Tuple ParseReplay(byte[] bytes, bool ignoreErrors = false, bool allowPTRRegion = false) + public static Tuple ParseReplay(byte[] bytes, ParseOptions parseOptions) { try { @@ -48,14 +49,13 @@ public static Tuple ParseReplay(byte[] bytes, bool ig // File in the version numbers for later use. MpqHeader.ParseHeader(replay, bytes); - if (!ignoreErrors && replay.ReplayBuild < 32455) + if (!parseOptions.IgnoreErrors && replay.ReplayBuild < 32455) return new Tuple(ReplayParseResult.PreAlphaWipe, null); - using (var memoryStream = new MemoryStream(bytes)) using (var archive = new MpqArchive(memoryStream)) - ParseReplayArchive(replay, archive, ignoreErrors); + ParseReplayArchive(replay, archive, parseOptions); - return ParseReplayResults(replay, ignoreErrors, allowPTRRegion); + return ParseReplayResults(replay, parseOptions.IgnoreErrors, parseOptions.AllowPTR); } catch { @@ -63,7 +63,8 @@ public static Tuple ParseReplay(byte[] bytes, bool ig } } - public static Tuple ParseReplay(string fileName, bool ignoreErrors, bool deleteFile, bool allowPTRRegion = false, bool detailedBattleLobbyParsing = false) + //public static Tuple ParseReplay(string fileName, bool ignoreErrors, bool deleteFile, bool allowPTRRegion = false, bool detailedBattleLobbyParsing = false) + public static Tuple ParseReplay(string fileName, bool deleteFile, ParseOptions parseOptions) { try { @@ -72,16 +73,16 @@ public static Tuple ParseReplay(string fileName, bool // File in the version numbers for later use. MpqHeader.ParseHeader(replay, fileName); - if (!ignoreErrors && replay.ReplayBuild < 32455) + if (!parseOptions.IgnoreErrors && replay.ReplayBuild < 32455) return new Tuple(ReplayParseResult.PreAlphaWipe, null); using (var archive = new MpqArchive(fileName)) - ParseReplayArchive(replay, archive, ignoreErrors, detailedBattleLobbyParsing); + ParseReplayArchive(replay, archive, parseOptions); if (deleteFile) File.Delete(fileName); - return ParseReplayResults(replay, ignoreErrors, allowPTRRegion); + return ParseReplayResults(replay, parseOptions.IgnoreErrors, parseOptions.AllowPTR); } catch { @@ -115,14 +116,14 @@ private static Tuple ParseReplayResults(Replay replay return new Tuple(ReplayParseResult.Success, replay); } - private static void ParseReplayArchive(Replay replay, MpqArchive archive, bool ignoreErrors, bool detailedBattleLobbyParsing = false) + private static void ParseReplayArchive(Replay replay, MpqArchive archive, ParseOptions parseOptions) { archive.AddListfileFilenames(); // Replay Details - ReplayDetails.Parse(replay, GetMpqFile(archive, ReplayDetails.FileName), ignoreErrors); + ReplayDetails.Parse(replay, GetMpqFile(archive, ReplayDetails.FileName), parseOptions.IgnoreErrors); - if (!ignoreErrors) + if (!parseOptions.IgnoreErrors) { if (replay.Players.Length != 10 || replay.Players.Count(i => i.IsWinner) != 5) // Filter out 'Try Me' games, any games without 10 players, and incomplete games @@ -140,61 +141,71 @@ private static void ParseReplayArchive(Replay replay, MpqArchive archive, bool i ReplayAttributeEvents.Parse(replay, GetMpqFile(archive, ReplayAttributeEvents.FileName)); - replay.TrackerEvents = ReplayTrackerEvents.Parse(GetMpqFile(archive, ReplayTrackerEvents.FileName)); - - try - { - replay.GameEvents = ReplayGameEvents.Parse(GetMpqFile(archive, ReplayGameEvents.FileName), replay.ClientListByUserID, replay.ReplayBuild, replay.ReplayVersionMajor); - replay.IsGameEventsParsedSuccessfully = true; - } - catch + if (parseOptions.ShouldParseEvents) { - replay.GameEvents = new List(); - } - - { - // Gather talent selections - var talentGameEventsDictionary = replay.GameEvents - .Where(i => i.eventType == GameEventType.CHeroTalentSelectedEvent) - .GroupBy(i => i.player) - .ToDictionary( - i => i.Key, - i => i.Select(j => new Talent { TalentID = (int)j.data.unsignedInt.Value, TimeSpanSelected = j.TimeSpan }).OrderBy(j => j.TimeSpanSelected).ToArray()); - - foreach (var player in talentGameEventsDictionary.Keys) - player.Talents = talentGameEventsDictionary[player]; - } - - // Replay Server Battlelobby - if (!ignoreErrors && archive.Any(i => i.Filename == ReplayServerBattlelobby.FileName)) - { - if (detailedBattleLobbyParsing) - ReplayServerBattlelobby.Parse(replay, GetMpqFile(archive, ReplayServerBattlelobby.FileName)); - else - ReplayServerBattlelobby.GetBattleTags(replay, GetMpqFile(archive, ReplayServerBattlelobby.FileName)); - } - - // Parse Unit Data using Tracker events - Unit.ParseUnitData(replay); - - // Parse Statistics - if (replay.ReplayBuild >= 40431) + //replay.TrackerEvents = ReplayTrackerEvents.Parse(GetMpqFile(archive, ReplayTrackerEvents.FileName)); try { - Statistics.Parse(replay); - replay.IsStatisticsParsedSuccessfully = true; + replay.GameEvents = ReplayGameEvents.Parse(GetMpqFile(archive, ReplayGameEvents.FileName), replay.ClientListByUserID, replay.ReplayBuild, replay.ReplayVersionMajor, parseOptions.ShouldParseMouseEvents); + replay.IsGameEventsParsedSuccessfully = true; } catch { - replay.IsGameEventsParsedSuccessfully = false; + replay.GameEvents = new List(); } - // Replay Message Events - // ReplayMessageEvents.Parse(replay, GetMpqFile(archive, ReplayMessageEvents.FileName)); + { + // Gather talent selections + var talentGameEventsDictionary = replay.GameEvents + .Where(i => i.eventType == GameEventType.CHeroTalentSelectedEvent) + .GroupBy(i => i.player) + .ToDictionary( + i => i.Key, + i => i.Select(j => new Talent { TalentID = (int)j.data.unsignedInt.Value, TimeSpanSelected = j.TimeSpan }).OrderBy(j => j.TimeSpanSelected).ToArray()); + + foreach (var player in talentGameEventsDictionary.Keys) + player.Talents = talentGameEventsDictionary[player]; + } + // Replay Server Battlelobby + if (!parseOptions.IgnoreErrors && archive.Any(i => i.Filename == ReplayServerBattlelobby.FileName)) + { + if (parseOptions.ShouldParseDetailedBattleLobby) + ReplayServerBattlelobby.Parse(replay, GetMpqFile(archive, ReplayServerBattlelobby.FileName)); + else + ReplayServerBattlelobby.GetBattleTags(replay, GetMpqFile(archive, ReplayServerBattlelobby.FileName)); + } + + // Parse Unit Data using Tracker events + if (parseOptions.ShouldParseUnits) + { + Unit.ParseUnitData(replay); + } - // Replay Resumable Events - // So far it doesn't look like this file has anything we would be interested in - // ReplayResumableEvents.Parse(replay, GetMpqFile(archive, "replay.resumable.events")); + // Parse Statistics + if (parseOptions.ShouldParseStatistics) + { + if (replay.ReplayBuild >= 40431) + try + { + Statistics.Parse(replay); + replay.IsStatisticsParsedSuccessfully = true; + } + catch + { + replay.IsGameEventsParsedSuccessfully = false; + } + } + + // Replay Message Events + if (parseOptions.ShouldParseMessageEvents) + { + ReplayMessageEvents.Parse(replay, GetMpqFile(archive, ReplayMessageEvents.FileName)); + } + + // Replay Resumable Events + // So far it doesn't look like this file has anything we would be interested in + // ReplayResumableEvents.Parse(replay, GetMpqFile(archive, "replay.resumable.events")); + } } public static byte[] GetMpqFile(MpqArchive archive, string fileName) diff --git a/Heroes.ReplayParser/MPQFiles/ReplayGameEvents.cs b/Heroes.ReplayParser/MPQFiles/ReplayGameEvents.cs index baed4eb..feff286 100644 --- a/Heroes.ReplayParser/MPQFiles/ReplayGameEvents.cs +++ b/Heroes.ReplayParser/MPQFiles/ReplayGameEvents.cs @@ -11,7 +11,7 @@ public class ReplayGameEvents { public const string FileName = "replay.game.events"; - public static List Parse(byte[] buffer, Player[] clientList, int replayBuild, int replayVersionMajor) + public static List Parse(byte[] buffer, Player[] clientList, int replayBuild, int replayVersionMajor, bool parseMouseMoveEvents) { // Referenced from https://raw.githubusercontent.com/Blizzard/heroprotocol/master/protocol39445.py @@ -40,7 +40,9 @@ public static List Parse(byte[] buffer, Player[] clientList, int repl case GameEventType.CUserFinishedLoadingSyncEvent: break; case GameEventType.CUserOptionsEvent: - gameEvent.data = new TrackerEventStructure { array = new[] { + gameEvent.data = new TrackerEventStructure + { + array = new[] { new TrackerEventStructure { unsignedInt = bitReader.Read(1) }, // m_gameFullyDownloaded new TrackerEventStructure { unsignedInt = bitReader.Read(1) }, // m_developmentCheatsEnabled new TrackerEventStructure { unsignedInt = bitReader.Read(1) }, // m_testCheatsEnabled @@ -54,7 +56,8 @@ public static List Parse(byte[] buffer, Player[] clientList, int repl new TrackerEventStructure { unsignedInt = bitReader.Read(32) }, // m_baseBuildNum new TrackerEventStructure { unsignedInt = bitReader.Read(32) }, // m_buildNum new TrackerEventStructure { unsignedInt = bitReader.Read(32) }, // m_versionFlags - new TrackerEventStructure { DataType = 2, blob = bitReader.ReadBlobPrecededWithLength(9) } /* m_hotkeyProfile */ } }; + new TrackerEventStructure { DataType = 2, blob = bitReader.ReadBlobPrecededWithLength(9) } /* m_hotkeyProfile */ } + }; break; case GameEventType.CBankFileEvent: gameEvent.data = new TrackerEventStructure { DataType = 2, blob = bitReader.ReadBlobPrecededWithLength(7) }; @@ -63,10 +66,13 @@ public static List Parse(byte[] buffer, Player[] clientList, int repl gameEvent.data = new TrackerEventStructure { DataType = 2, blob = bitReader.ReadBlobPrecededWithLength(6) }; break; case GameEventType.CBankKeyEvent: - gameEvent.data = new TrackerEventStructure { array = new[] { + gameEvent.data = new TrackerEventStructure + { + array = new[] { new TrackerEventStructure { DataType = 2, blob = bitReader.ReadBlobPrecededWithLength(6) }, new TrackerEventStructure { unsignedInt = bitReader.Read(32) }, - new TrackerEventStructure { DataType = 2, blob = bitReader.ReadBlobPrecededWithLength(7) } } }; + new TrackerEventStructure { DataType = 2, blob = bitReader.ReadBlobPrecededWithLength(7) } } + }; break; case GameEventType.CBankSignatureEvent: gameEvent.data = new TrackerEventStructure { DataType = 2, array = new TrackerEventStructure[bitReader.Read(5)] }; @@ -94,7 +100,8 @@ public static List Parse(byte[] buffer, Player[] clientList, int repl gameEvent.data.array[0] = new TrackerEventStructure { array = new[] { new TrackerEventStructure { unsignedInt = bitReader.Read(20) }, new TrackerEventStructure { unsignedInt = bitReader.Read(20) }, new TrackerEventStructure { vInt = bitReader.Read(32) - 2147483648 } } }; break; case 2: // TargetUnit - gameEvent.data.array[0] = new TrackerEventStructure { + gameEvent.data.array[0] = new TrackerEventStructure + { array = new[] { new TrackerEventStructure { unsignedInt = bitReader.Read(16) }, // m_targetUnitFlags new TrackerEventStructure { unsignedInt = bitReader.Read(8) }, // m_timer @@ -102,7 +109,8 @@ public static List Parse(byte[] buffer, Player[] clientList, int repl new TrackerEventStructure { unsignedInt = bitReader.Read(16) }, // m_snapshotUnitLink new TrackerEventStructure(), new TrackerEventStructure(), - new TrackerEventStructure(), } }; + new TrackerEventStructure(), } + }; if (bitReader.ReadBoolean()) // m_snapshotControlPlayerId gameEvent.data.array[0].array[4].unsignedInt = bitReader.Read(4); @@ -135,26 +143,28 @@ public static List Parse(byte[] buffer, Player[] clientList, int repl gameEvent.data.array[0] = new TrackerEventStructure { array = new TrackerEventStructure[24] }; else if (replayBuild <= 45635) gameEvent.data.array[0] = new TrackerEventStructure { array = new TrackerEventStructure[26] }; - else if(replayVersionMajor < 2) - gameEvent.data.array[0] = new TrackerEventStructure { array = new TrackerEventStructure[25] }; - else if (replayBuild < 59837 || replayBuild == 59988) - gameEvent.data.array[0] = new TrackerEventStructure { array = new TrackerEventStructure[26] }; - else if (replayBuild < 62833) - gameEvent.data.array[0] = new TrackerEventStructure { array = new TrackerEventStructure[27] }; - else - gameEvent.data.array[0] = new TrackerEventStructure { array = new TrackerEventStructure[26] }; - - for (var i = 0; i < gameEvent.data.array[0].array.Length; i++) + else if (replayVersionMajor < 2) + gameEvent.data.array[0] = new TrackerEventStructure { array = new TrackerEventStructure[25] }; + else if (replayBuild < 59837 || replayBuild == 59988 || replayBuild > 62424) + gameEvent.data.array[0] = new TrackerEventStructure { array = new TrackerEventStructure[26] }; + else if (replayBuild < 62833) + gameEvent.data.array[0] = new TrackerEventStructure { array = new TrackerEventStructure[27] }; + else + gameEvent.data.array[0] = new TrackerEventStructure { array = new TrackerEventStructure[26] }; + + for (var i = 0; i < gameEvent.data.array[0].array.Length; i++) gameEvent.data.array[0].array[i] = new TrackerEventStructure { DataType = 7, unsignedInt = bitReader.Read(1) }; // m_abil if (bitReader.ReadBoolean()) { - gameEvent.data.array[1] = new TrackerEventStructure { + gameEvent.data.array[1] = new TrackerEventStructure + { array = new[] { new TrackerEventStructure { unsignedInt = bitReader.Read(16) }, // m_abilLink new TrackerEventStructure { unsignedInt = bitReader.Read(5) }, // m_abilCmdIndex - new TrackerEventStructure() } }; + new TrackerEventStructure() } + }; if (bitReader.ReadBoolean()) // m_abilCmdData gameEvent.data.array[1].array[2].unsignedInt = bitReader.Read(8); @@ -169,14 +179,17 @@ public static List Parse(byte[] buffer, Player[] clientList, int repl gameEvent.data.array[2] = new TrackerEventStructure { array = new[] { new TrackerEventStructure { unsignedInt = bitReader.Read(20) }, new TrackerEventStructure { unsignedInt = bitReader.Read(20) }, new TrackerEventStructure { vInt = bitReader.Read(32) - 2147483648 } } }; break; case 2: // TargetUnit - gameEvent.data.array[2] = new TrackerEventStructure { array = new[] { + gameEvent.data.array[2] = new TrackerEventStructure + { + array = new[] { new TrackerEventStructure { unsignedInt = bitReader.Read(16) }, // m_targetUnitFlags new TrackerEventStructure { unsignedInt = bitReader.Read(8) }, // m_timer new TrackerEventStructure { unsignedInt = bitReader.Read(32) }, // m_tag new TrackerEventStructure { unsignedInt = bitReader.Read(16) }, // m_snapshotUnitLink new TrackerEventStructure(), new TrackerEventStructure(), - new TrackerEventStructure(), } }; + new TrackerEventStructure(), } + }; if (bitReader.ReadBoolean()) // m_snapshotControlPlayerId gameEvent.data.array[2].array[4].unsignedInt = bitReader.Read(4); @@ -204,15 +217,18 @@ public static List Parse(byte[] buffer, Player[] clientList, int repl gameEvent.data.array[4] = new TrackerEventStructure { unsignedInt = bitReader.Read(32) }; // m_unitGroup break; case GameEventType.CSelectionDeltaEvent: - gameEvent.data = new TrackerEventStructure { array = new[] { - new TrackerEventStructure { unsignedInt = bitReader.Read(4) }, // m_controlGroupId - - // m_delta - new TrackerEventStructure { array = new[] { - new TrackerEventStructure { unsignedInt = bitReader.Read(replayVersionMajor < 2 ? 9 : 5) }, // m_subgroupIndex - new TrackerEventStructure(), - new TrackerEventStructure(), - new TrackerEventStructure() } } } }; + gameEvent.data = new TrackerEventStructure + { + array = new[] { + new TrackerEventStructure { unsignedInt = bitReader.Read(4) }, // m_controlGroupId + + // m_delta + new TrackerEventStructure { array = new[] { + new TrackerEventStructure { unsignedInt = bitReader.Read(replayVersionMajor < 2 ? 9 : 5) }, // m_subgroupIndex + new TrackerEventStructure(), + new TrackerEventStructure(), + new TrackerEventStructure() } } } + }; // m_removeMask switch (bitReader.Read(2)) @@ -233,11 +249,14 @@ public static List Parse(byte[] buffer, Player[] clientList, int repl // m_addSubgroups gameEvent.data.array[1].array[2] = new TrackerEventStructure { array = new TrackerEventStructure[bitReader.Read(replayVersionMajor < 2 ? 9 : 6)] }; for (var i = 0; i < gameEvent.data.array[1].array[2].array.Length; i++) - gameEvent.data.array[1].array[2].array[i] = new TrackerEventStructure { array = new[] { + gameEvent.data.array[1].array[2].array[i] = new TrackerEventStructure + { + array = new[] { new TrackerEventStructure { unsignedInt = bitReader.Read(16) }, // m_unitLink new TrackerEventStructure { unsignedInt = bitReader.Read(8) }, // m_subgroupPriority new TrackerEventStructure { unsignedInt = bitReader.Read(8) }, // m_intraSubgroupPriority - new TrackerEventStructure { unsignedInt = bitReader.Read(replayVersionMajor < 2 ? 9 : 6) } } }; // m_count + new TrackerEventStructure { unsignedInt = bitReader.Read(replayVersionMajor < 2 ? 9 : 6) } } + }; // m_count // m_addUnitTags gameEvent.data.array[1].array[3] = new TrackerEventStructure { array = new TrackerEventStructure[bitReader.Read(replayVersionMajor < 2 ? 9 : 6)] }; @@ -253,8 +272,8 @@ public static List Parse(byte[] buffer, Player[] clientList, int repl else bitReader.Read(3); - // m_mask - switch(bitReader.Read(2)) + // m_mask + switch (bitReader.Read(2)) { case 0: // None break; @@ -273,19 +292,19 @@ public static List Parse(byte[] buffer, Player[] clientList, int repl bitReader.Read(4); // m_controlGroupId // m_selectionSyncData - if (replayVersionMajor < 2) - { - bitReader.Read(9); // m_count - bitReader.Read(9); // m_subgroupCount - bitReader.Read(9); // m_activeSubgroupIndex - } - else - { - bitReader.Read(6); // m_count - bitReader.Read(6); // m_subgroupCount - bitReader.Read(5); // m_activeSubgroupIndex - } - + if (replayVersionMajor < 2) + { + bitReader.Read(9); // m_count + bitReader.Read(9); // m_subgroupCount + bitReader.Read(9); // m_activeSubgroupIndex + } + else + { + bitReader.Read(6); // m_count + bitReader.Read(6); // m_subgroupCount + bitReader.Read(5); // m_activeSubgroupIndex + } + bitReader.Read(32); // m_unitTagsChecksum bitReader.Read(32); // m_subgroupIndicesChecksum bitReader.Read(32); // m_subgroupsChecksum @@ -313,12 +332,15 @@ public static List Parse(byte[] buffer, Player[] clientList, int repl bitReader.Read(3); // m_speed break; case GameEventType.CTriggerPingEvent: - gameEvent.data = new TrackerEventStructure { array = new[] { + gameEvent.data = new TrackerEventStructure + { + array = new[] { new TrackerEventStructure { vInt = bitReader.Read(32) - 2147483648 }, new TrackerEventStructure { vInt = bitReader.Read(32) - 2147483648 }, new TrackerEventStructure { unsignedInt = bitReader.Read(32) }, new TrackerEventStructure { unsignedInt = bitReader.Read(1) }, - new TrackerEventStructure { vInt = bitReader.Read(32) - 2147483648 } } }; + new TrackerEventStructure { vInt = bitReader.Read(32) - 2147483648 } } + }; break; case GameEventType.CUnitClickEvent: gameEvent.data = new TrackerEventStructure { unsignedInt = bitReader.Read(32) }; // m_unitTag @@ -362,10 +384,13 @@ public static List Parse(byte[] buffer, Player[] clientList, int repl bitReader.Read(32); // m_difficultyLevel, offset -2147483648 break; case GameEventType.CTriggerDialogControlEvent: - gameEvent.data = new TrackerEventStructure { array = new[] { + gameEvent.data = new TrackerEventStructure + { + array = new[] { new TrackerEventStructure { vInt = bitReader.Read(32) /* Actually signed - not handled correctly */ }, new TrackerEventStructure { vInt = bitReader.Read(32) /* Actually signed - not handled correctly */ }, - new TrackerEventStructure() } }; + new TrackerEventStructure() } + }; switch (bitReader.Read(3)) { case 0: // None @@ -384,14 +409,14 @@ public static List Parse(byte[] buffer, Player[] clientList, int repl gameEvent.data.array[2].blob = bitReader.ReadBlobPrecededWithLength(11); break; case 5: // MouseButton or MouseEvent - if(replayBuild == 57547 || replayBuild > 57589) - gameEvent.data.array[2].array = new[] - { - new TrackerEventStructure { vInt = bitReader.Read(16) }, // m_button - new TrackerEventStructure { vInt = bitReader.Read(16) } // m_metaKeyFlags - }; - else - gameEvent.data.array[2].unsignedInt = bitReader.Read(32); + if (replayBuild == 57547 || replayBuild > 57589) + gameEvent.data.array[2].array = new[] + { + new TrackerEventStructure { vInt = bitReader.Read(16) }, // m_button + new TrackerEventStructure { vInt = bitReader.Read(16) } // m_metaKeyFlags + }; + else + gameEvent.data.array[2].unsignedInt = bitReader.Read(32); break; } break; @@ -408,26 +433,36 @@ public static List Parse(byte[] buffer, Player[] clientList, int repl gameEvent.data = new TrackerEventStructure { unsignedInt = bitReader.Read(1) }; break; case GameEventType.CTriggerMouseClickedEvent: - bitReader.Read(32); // m_button - bitReader.ReadBoolean(); // m_down - bitReader.Read(11); // m_posUI X - bitReader.Read(11); // m_posUI Y - bitReader.Read(20); // m_posWorld X - bitReader.Read(20); // m_posWorld Y - bitReader.Read(32); // m_posWorld Z (Offset -2147483648) - bitReader.Read(8); // m_flags (-128) + bitReader.Skip(32 + 1 + 11 + 11 + 20 + 20 + 32 + 8); + //bitReader.Read(32); // m_button + //bitReader.ReadBoolean(); // m_down + //bitReader.Read(11); // m_posUI X + //bitReader.Read(11); // m_posUI Y + //bitReader.Read(20); // m_posWorld X + //bitReader.Read(20); // m_posWorld Y + //bitReader.Read(32); // m_posWorld Z (Offset -2147483648) + //bitReader.Read(8); // m_flags (-128) break; case GameEventType.CTriggerMouseMovedEvent: - gameEvent.data = new TrackerEventStructure { array = new[] { - // m_posUI - new TrackerEventStructure { unsignedInt = bitReader.Read(11) }, - new TrackerEventStructure { unsignedInt = bitReader.Read(11) }, - - // m_posWorld - new TrackerEventStructure { array = new[] { new TrackerEventStructure { unsignedInt = bitReader.Read(20) }, new TrackerEventStructure { unsignedInt = bitReader.Read(20) }, new TrackerEventStructure { vInt = bitReader.Read(32) - 2147483648 } } }, - - // m_flags - new TrackerEventStructure { vInt = bitReader.Read(8) - 128 } } }; + if (parseMouseMoveEvents) + { + gameEvent.data = new TrackerEventStructure + { + array = new[] { + // m_posUI + new TrackerEventStructure { unsignedInt = bitReader.Read(11) }, + new TrackerEventStructure { unsignedInt = bitReader.Read(11) }, + // m_posWorld + new TrackerEventStructure { array = new[] { new TrackerEventStructure { unsignedInt = bitReader.Read(20) }, new TrackerEventStructure { unsignedInt = bitReader.Read(20) }, new TrackerEventStructure { vInt = bitReader.Read(32) - 2147483648 } } }, + // m_flags + new TrackerEventStructure { vInt = bitReader.Read(8) - 128 } + } + }; + } + else + { + bitReader.Skip(11 + 11 + 20 + 20 + 32 + 8); + } break; case GameEventType.CTriggerHotkeyPressedEvent: gameEvent.data = new TrackerEventStructure { unsignedInt = bitReader.Read(32) }; // May be missing an offset value @@ -452,11 +487,11 @@ public static List Parse(byte[] buffer, Player[] clientList, int repl gameEvent.data = new TrackerEventStructure { vInt = bitReader.Read(32) - 2147483648 }; break; case GameEventType.CGameUserLeaveEvent: - // m_leaveReason - if(replayBuild >= 55929) - bitReader.Read(5); - else - bitReader.Read(4); + // m_leaveReason + if (replayBuild >= 55929) + bitReader.Read(5); + else + bitReader.Read(4); break; case GameEventType.CGameUserJoinEvent: gameEvent.data = new TrackerEventStructure { array = new TrackerEventStructure[5] }; diff --git a/Heroes.ReplayParser/ParseOptions.cs b/Heroes.ReplayParser/ParseOptions.cs new file mode 100644 index 0000000..191a733 --- /dev/null +++ b/Heroes.ReplayParser/ParseOptions.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Heroes.ReplayParser +{ + /// + /// Parse options, represented as a class. + /// + /// Default options are automatically set, typical use would + /// be to either use one of the provided static option sets, + /// or to set your own option set in the initializer + /// + /// i.e. new ParseOption() { + /// ShouldParseUnits = true + /// } + /// + public class ParseOptions + { + public bool IgnoreErrors { get; set; } = false; + public bool AllowPTR { get; set; } = false; + public bool ShouldParseEvents { get; set; } = true; + public bool ShouldParseUnits { get; set; } = false; + public bool ShouldParseMouseEvents { get; set; } = false; + public bool ShouldParseDetailedBattleLobby { get; set; } = true; + public bool ShouldParseMessageEvents { get; set; } = false; + public bool ShouldParseStatistics { get; set; } = true; + + /// + /// Parsing with all options enabled + /// + public static ParseOptions FullParsing => new ParseOptions() + { + AllowPTR = true, + ShouldParseDetailedBattleLobby = true, + ShouldParseMouseEvents = true, + ShouldParseUnits = true, + ShouldParseMessageEvents = true, + }; + + /// + /// Parse as little as possible + /// + public static ParseOptions MinimalParsing => new ParseOptions() + { + ShouldParseEvents = false, + ShouldParseDetailedBattleLobby = false + }; + + /// + /// Parsing for typical needs, excludes events, units and mouseevents. + /// + public static ParseOptions TypicalParsing => new ParseOptions(); + + } +} From d7946dc5c5e5ddca604ac9e051245d5a3cbada0f Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 21 Jan 2020 10:58:49 -0500 Subject: [PATCH 2/8] fix: added tracker events back in --- Heroes.ReplayParser/DataParser.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Heroes.ReplayParser/DataParser.cs b/Heroes.ReplayParser/DataParser.cs index bfdf2fb..a48c2ae 100644 --- a/Heroes.ReplayParser/DataParser.cs +++ b/Heroes.ReplayParser/DataParser.cs @@ -38,8 +38,6 @@ public enum ReplayParseResult { "Towers of Doom", new Tuple(4.0, 42.0, 1.03, 0.925) }, { "Battlefield of Eternity", new Tuple(-5.0, 33.0, 1.09, 0.96) } }; - - //public static Tuple ParseReplay(byte[] bytes, bool ignoreErrors = false, bool allowPTRRegion = false) public static Tuple ParseReplay(byte[] bytes, ParseOptions parseOptions) { try @@ -143,7 +141,7 @@ private static void ParseReplayArchive(Replay replay, MpqArchive archive, ParseO if (parseOptions.ShouldParseEvents) { - //replay.TrackerEvents = ReplayTrackerEvents.Parse(GetMpqFile(archive, ReplayTrackerEvents.FileName)); + replay.TrackerEvents = ReplayTrackerEvents.Parse(GetMpqFile(archive, ReplayTrackerEvents.FileName)); try { replay.GameEvents = ReplayGameEvents.Parse(GetMpqFile(archive, ReplayGameEvents.FileName), replay.ClientListByUserID, replay.ReplayBuild, replay.ReplayVersionMajor, parseOptions.ShouldParseMouseEvents); From b1cc7387b6ae884caeaf26eaf7c5b184141c5df0 Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 21 Jan 2020 11:36:53 -0500 Subject: [PATCH 3/8] Fix: Resolved conflict on program.cs --- Heroes.ReplayParser.ConsoleApplication/Program.cs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Heroes.ReplayParser.ConsoleApplication/Program.cs b/Heroes.ReplayParser.ConsoleApplication/Program.cs index 8d8177c..25b8262 100644 --- a/Heroes.ReplayParser.ConsoleApplication/Program.cs +++ b/Heroes.ReplayParser.ConsoleApplication/Program.cs @@ -15,13 +15,11 @@ static void Main(string[] args) // Attempt to parse the replay // Ignore errors can be set to true if you want to attempt to parse currently unsupported replays, such as 'VS AI' or 'PTR Region' replays - var replayParseResult = DataParser.ParseReplay(randomReplayFileName, deleteFile: false, ParseOptions.TypicalParsing); + var (replayParseResult, replay) = DataParser.ParseReplay(randomReplayFileName, deleteFile: false, ParseOptions.TypicalParsing); // If successful, the Replay object now has all currently available information - if (replayParseResult.Item1 == DataParser.ReplayParseResult.Success) + if (replayParseResult == DataParser.ReplayParseResult.Success) { - var replay = replayParseResult.Item2; - Console.WriteLine("Replay Build: " + replay.ReplayBuild); Console.WriteLine("Map: " + replay.Map); foreach (var player in replay.Players.OrderByDescending(i => i.IsWinner)) @@ -30,7 +28,7 @@ static void Main(string[] args) Console.WriteLine("Press Any Key to Close"); } else - Console.WriteLine("Failed to Parse Replay: " + replayParseResult.Item1); + Console.WriteLine("Failed to Parse Replay: " + replayParseResult); Console.Read(); } From 52c06a2fee42980b73f16915567f83cbd2615047 Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 21 Jan 2020 11:38:29 -0500 Subject: [PATCH 4/8] fix: resolved conflict on BitReader.cs file --- Heroes.ReplayParser/BitReader.cs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Heroes.ReplayParser/BitReader.cs b/Heroes.ReplayParser/BitReader.cs index cbb92c4..f2b8e58 100644 --- a/Heroes.ReplayParser/BitReader.cs +++ b/Heroes.ReplayParser/BitReader.cs @@ -28,16 +28,13 @@ public BitReader(Stream stream) /// public int Cursor { get; private set; } + /// /// Gets a value indicating whether the end of stream has been reached. /// - public bool EndOfStream - { - get - { - return (Cursor >> 3) == stream.Length; - } - } + public bool EndOfStream => (Cursor >> 3) == stream.Length; + + /// /// Reads up to 32 bits from the stream, returning them as a uint. From 9011e36c3548f5d11516308ff1e5843a93b8a041 Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 21 Jan 2020 11:40:28 -0500 Subject: [PATCH 5/8] fix: Resolved conflict on ReplayGameEvents.cs file --- Heroes.ReplayParser/MPQFiles/ReplayGameEvents.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Heroes.ReplayParser/MPQFiles/ReplayGameEvents.cs b/Heroes.ReplayParser/MPQFiles/ReplayGameEvents.cs index feff286..bde5f8d 100644 --- a/Heroes.ReplayParser/MPQFiles/ReplayGameEvents.cs +++ b/Heroes.ReplayParser/MPQFiles/ReplayGameEvents.cs @@ -488,10 +488,7 @@ public static List Parse(byte[] buffer, Player[] clientList, int repl break; case GameEventType.CGameUserLeaveEvent: // m_leaveReason - if (replayBuild >= 55929) - bitReader.Read(5); - else - bitReader.Read(4); + bitReader.Read(replayBuild >= 55929 ? 5 : 4); break; case GameEventType.CGameUserJoinEvent: gameEvent.data = new TrackerEventStructure { array = new TrackerEventStructure[5] }; From b4a4373e7f980fa1f9f57c7ee4d4c44e3a2f8720 Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 21 Jan 2020 11:41:22 -0500 Subject: [PATCH 6/8] feat: Switched to Full Parsing as default --- Heroes.ReplayParser.ConsoleApplication/Program.cs | 2 +- Heroes.ReplayParser/BitReader.cs | 2 -- Heroes.ReplayParser/DataParser.cs | 1 + 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Heroes.ReplayParser.ConsoleApplication/Program.cs b/Heroes.ReplayParser.ConsoleApplication/Program.cs index 25b8262..6146387 100644 --- a/Heroes.ReplayParser.ConsoleApplication/Program.cs +++ b/Heroes.ReplayParser.ConsoleApplication/Program.cs @@ -15,7 +15,7 @@ static void Main(string[] args) // Attempt to parse the replay // Ignore errors can be set to true if you want to attempt to parse currently unsupported replays, such as 'VS AI' or 'PTR Region' replays - var (replayParseResult, replay) = DataParser.ParseReplay(randomReplayFileName, deleteFile: false, ParseOptions.TypicalParsing); + var (replayParseResult, replay) = DataParser.ParseReplay(randomReplayFileName, deleteFile: false, ParseOptions.FullParsing); // If successful, the Replay object now has all currently available information if (replayParseResult == DataParser.ReplayParseResult.Success) diff --git a/Heroes.ReplayParser/BitReader.cs b/Heroes.ReplayParser/BitReader.cs index f2b8e58..9beeb4c 100644 --- a/Heroes.ReplayParser/BitReader.cs +++ b/Heroes.ReplayParser/BitReader.cs @@ -34,8 +34,6 @@ public BitReader(Stream stream) /// public bool EndOfStream => (Cursor >> 3) == stream.Length; - - /// /// Reads up to 32 bits from the stream, returning them as a uint. /// diff --git a/Heroes.ReplayParser/DataParser.cs b/Heroes.ReplayParser/DataParser.cs index a48c2ae..bc091bc 100644 --- a/Heroes.ReplayParser/DataParser.cs +++ b/Heroes.ReplayParser/DataParser.cs @@ -150,6 +150,7 @@ private static void ParseReplayArchive(Replay replay, MpqArchive archive, ParseO catch { replay.GameEvents = new List(); + } { From ad81f189927466497706bf44529daa226a60bdb3 Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 21 Jan 2020 11:49:24 -0500 Subject: [PATCH 7/8] fix: cleanup --- Heroes.ReplayParser/DataParser.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/Heroes.ReplayParser/DataParser.cs b/Heroes.ReplayParser/DataParser.cs index 9d0052a..e0d5689 100644 --- a/Heroes.ReplayParser/DataParser.cs +++ b/Heroes.ReplayParser/DataParser.cs @@ -62,8 +62,6 @@ public static Tuple ParseReplay(byte[] bytes, ParseOp return new Tuple(ReplayParseResult.Exception, null); } } - - //public static Tuple ParseReplay(string fileName, bool ignoreErrors, bool deleteFile, bool allowPTRRegion = false, bool detailedBattleLobbyParsing = false) public static Tuple ParseReplay(string fileName, bool deleteFile, ParseOptions parseOptions) { try From 78b77c16910e0fcedbdba3ce284ef57bf4ddd6ef Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 21 Jan 2020 12:26:05 -0500 Subject: [PATCH 8/8] fix: Updated ParseOptions to use original Default options Updated ParseOptions to use original Default options and also set the default parse to DefaultParsing --- .../Program.cs | 2 +- Heroes.ReplayParser/ParseOptions.cs | 23 +++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/Heroes.ReplayParser.ConsoleApplication/Program.cs b/Heroes.ReplayParser.ConsoleApplication/Program.cs index 2bc7192..d93e4f5 100644 --- a/Heroes.ReplayParser.ConsoleApplication/Program.cs +++ b/Heroes.ReplayParser.ConsoleApplication/Program.cs @@ -14,7 +14,7 @@ static void Main(string[] args) // Attempt to parse the replay // Ignore errors can be set to true if you want to attempt to parse currently unsupported replays, such as 'VS AI' or 'PTR Region' replays - var (replayParseResult, replay) = DataParser.ParseReplay(randomReplayFileName, deleteFile: false, ParseOptions.FullParsing); + var (replayParseResult, replay) = DataParser.ParseReplay(randomReplayFileName, deleteFile: false, ParseOptions.DefaultParsing); // If successful, the Replay object now has all currently available information if (replayParseResult == DataParser.ReplayParseResult.Success) diff --git a/Heroes.ReplayParser/ParseOptions.cs b/Heroes.ReplayParser/ParseOptions.cs index 191a733..8e4d8a0 100644 --- a/Heroes.ReplayParser/ParseOptions.cs +++ b/Heroes.ReplayParser/ParseOptions.cs @@ -20,9 +20,9 @@ public class ParseOptions public bool IgnoreErrors { get; set; } = false; public bool AllowPTR { get; set; } = false; public bool ShouldParseEvents { get; set; } = true; - public bool ShouldParseUnits { get; set; } = false; - public bool ShouldParseMouseEvents { get; set; } = false; - public bool ShouldParseDetailedBattleLobby { get; set; } = true; + public bool ShouldParseUnits { get; set; } = true; + public bool ShouldParseMouseEvents { get; set; } = true; + public bool ShouldParseDetailedBattleLobby { get; set; } = false; public bool ShouldParseMessageEvents { get; set; } = false; public bool ShouldParseStatistics { get; set; } = true; @@ -34,23 +34,36 @@ public class ParseOptions AllowPTR = true, ShouldParseDetailedBattleLobby = true, ShouldParseMouseEvents = true, - ShouldParseUnits = true, ShouldParseMessageEvents = true, }; + /// + /// Parse as excluding Units, Mouse Events, and Enabling the DetailedBattleLobby + /// + public static ParseOptions MediumParsing => new ParseOptions() + { + ShouldParseUnits = false, + ShouldParseMouseEvents = false, + ShouldParseDetailedBattleLobby = true, + }; + /// /// Parse as little as possible /// public static ParseOptions MinimalParsing => new ParseOptions() { ShouldParseEvents = false, + ShouldParseUnits = false, + ShouldParseMouseEvents = false, ShouldParseDetailedBattleLobby = false }; + + /// /// Parsing for typical needs, excludes events, units and mouseevents. /// - public static ParseOptions TypicalParsing => new ParseOptions(); + public static ParseOptions DefaultParsing => new ParseOptions(); } }